diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 0000000000..2b0aa21219 --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +8.2.1 diff --git a/.clang-tidy b/.clang-tidy index 56938a598d..d6cd768f6d 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,7 +1,37 @@ --- -Checks: 'clang-analyzer-*,readability-redundant-*,performance-*' -WarningsAsErrors: 'clang-analyzer-*,readability-redundant-*,performance-*' -HeaderFilterRegex: '.*' -AnalyzeTemporaryDtors: false +Checks: > + abseil-*, + bugprone-*, + clang-analyzer-*, + cppcoreguidelines-*, + google-*, + misc-*, + performance-*, + readability-*, + -clang-analyzer-deadcode*, + -clang-analyzer-optin*, + -readability-identifier-length +WarningsAsErrors: '' +HeaderFilterRegex: '' FormatStyle: none -User: user +CheckOptions: + llvm-else-after-return.WarnOnConditionVariables: 'false' + modernize-loop-convert.MinConfidence: reasonable + modernize-replace-auto-ptr.IncludeStyle: llvm + cert-str34-c.DiagnoseSignedUnsignedCharComparisons: 'false' + google-readability-namespace-comments.ShortNamespaceLines: '10' + cert-err33-c.CheckedFunctions: '::aligned_alloc;::asctime_s;::at_quick_exit;::atexit;::bsearch;::bsearch_s;::btowc;::c16rtomb;::c32rtomb;::calloc;::clock;::cnd_broadcast;::cnd_init;::cnd_signal;::cnd_timedwait;::cnd_wait;::ctime_s;::fclose;::fflush;::fgetc;::fgetpos;::fgets;::fgetwc;::fopen;::fopen_s;::fprintf;::fprintf_s;::fputc;::fputs;::fputwc;::fputws;::fread;::freopen;::freopen_s;::fscanf;::fscanf_s;::fseek;::fsetpos;::ftell;::fwprintf;::fwprintf_s;::fwrite;::fwscanf;::fwscanf_s;::getc;::getchar;::getenv;::getenv_s;::gets_s;::getwc;::getwchar;::gmtime;::gmtime_s;::localtime;::localtime_s;::malloc;::mbrtoc16;::mbrtoc32;::mbsrtowcs;::mbsrtowcs_s;::mbstowcs;::mbstowcs_s;::memchr;::mktime;::mtx_init;::mtx_lock;::mtx_timedlock;::mtx_trylock;::mtx_unlock;::printf_s;::putc;::putwc;::raise;::realloc;::remove;::rename;::scanf;::scanf_s;::setlocale;::setvbuf;::signal;::snprintf;::snprintf_s;::sprintf;::sprintf_s;::sscanf;::sscanf_s;::strchr;::strerror_s;::strftime;::strpbrk;::strrchr;::strstr;::strtod;::strtof;::strtoimax;::strtok;::strtok_s;::strtol;::strtold;::strtoll;::strtoul;::strtoull;::strtoumax;::strxfrm;::swprintf;::swprintf_s;::swscanf;::swscanf_s;::thrd_create;::thrd_detach;::thrd_join;::thrd_sleep;::time;::timespec_get;::tmpfile;::tmpfile_s;::tmpnam;::tmpnam_s;::tss_create;::tss_get;::tss_set;::ungetc;::ungetwc;::vfprintf;::vfprintf_s;::vfscanf;::vfscanf_s;::vfwprintf;::vfwprintf_s;::vfwscanf;::vfwscanf_s;::vprintf_s;::vscanf;::vscanf_s;::vsnprintf;::vsnprintf_s;::vsprintf;::vsprintf_s;::vsscanf;::vsscanf_s;::vswprintf;::vswprintf_s;::vswscanf;::vswscanf_s;::vwprintf_s;::vwscanf;::vwscanf_s;::wcrtomb;::wcschr;::wcsftime;::wcspbrk;::wcsrchr;::wcsrtombs;::wcsrtombs_s;::wcsstr;::wcstod;::wcstof;::wcstoimax;::wcstok;::wcstok_s;::wcstol;::wcstold;::wcstoll;::wcstombs;::wcstombs_s;::wcstoul;::wcstoull;::wcstoumax;::wcsxfrm;::wctob;::wctrans;::wctype;::wmemchr;::wprintf_s;::wscanf;::wscanf_s;' + cert-oop54-cpp.WarnOnlyIfThisHasSuspiciousField: 'false' + cert-dcl16-c.NewSuffixes: 'L;LL;LU;LLU' + google-readability-braces-around-statements.ShortStatementLines: '1' + cppcoreguidelines-non-private-member-variables-in-classes.IgnoreClassesWithAllMemberVariablesBeingPublic: 'true' + google-readability-namespace-comments.SpacesBeforeComments: '2' + modernize-loop-convert.MaxCopySize: '16' + modernize-pass-by-value.IncludeStyle: llvm + modernize-use-nullptr.NullMacros: 'NULL' + llvm-qualified-auto.AddConstToQualified: 'false' + modernize-loop-convert.NamingStyle: CamelCase + llvm-else-after-return.WarnOnUnfixable: 'false' + google-readability-function-size.StatementThreshold: '800' +... + diff --git a/.clang-tidy.ignore b/.clang-tidy.ignore new file mode 100644 index 0000000000..dba559d6ca --- /dev/null +++ b/.clang-tidy.ignore @@ -0,0 +1 @@ +.*third_party/.* diff --git a/.github/.libcxx-setup.sh b/.github/.libcxx-setup.sh deleted file mode 100755 index 56008403ae..0000000000 --- a/.github/.libcxx-setup.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -# Checkout LLVM sources -git clone --depth=1 https://github.com/llvm/llvm-project.git llvm-project - -# Setup libc++ options -if [ -z "$BUILD_32_BITS" ]; then - export BUILD_32_BITS=OFF && echo disabling 32 bit build -fi - -# Build and install libc++ (Use unstable ABI for better sanitizer coverage) -cd ./llvm-project -cmake -DCMAKE_C_COMPILER=${C_COMPILER} \ - -DCMAKE_CXX_COMPILER=${COMPILER} \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DLIBCXX_ABI_UNSTABLE=OFF \ - -DLLVM_USE_SANITIZER=${LIBCXX_SANITIZER} \ - -DLLVM_BUILD_32_BITS=${BUILD_32_BITS} \ - -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' \ - -S llvm -B llvm-build -G "Unix Makefiles" -make -C llvm-build -j3 cxx cxxabi -sudo make -C llvm-build install-cxx install-cxxabi -cd .. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..3661e978bb --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: daily + + - package-ecosystem: pip + directory: /tools + schedule: + interval: daily diff --git a/.github/install_bazel.sh b/.github/install_bazel.sh new file mode 100644 index 0000000000..a1693b87ea --- /dev/null +++ b/.github/install_bazel.sh @@ -0,0 +1,12 @@ +if ! bazel version; then + arch=$(uname -m) + if [ "$arch" == "aarch64" ]; then + arch="arm64" + fi + echo "Downloading $arch Bazel binary from GitHub releases." + curl -L -o $HOME/bin/bazel --create-dirs "https://github.com/bazelbuild/bazel/releases/download/8.2.0/bazel-8.2.0-linux-$arch" + chmod +x $HOME/bin/bazel +else + # Bazel is installed for the correct architecture + exit 0 +fi diff --git a/.github/libcxx-setup.sh b/.github/libcxx-setup.sh new file mode 100755 index 0000000000..8966194d59 --- /dev/null +++ b/.github/libcxx-setup.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -e + +# Checkout LLVM sources +git clone --filter=blob:none --depth=1 --branch llvmorg-19.1.6 --no-checkout https://github.com/llvm/llvm-project.git llvm-project +cd llvm-project +git sparse-checkout set --cone +git checkout llvmorg-19.1.6 +git sparse-checkout set cmake llvm/cmake runtimes libcxx libcxxabi +cd .. + +## Setup libc++ options +if [ -z "$BUILD_32_BITS" ]; then + export BUILD_32_BITS=OFF && echo disabling 32 bit build +fi + +## Build and install libc++ (Use unstable ABI for better sanitizer coverage) +mkdir llvm-build && cd llvm-build +cmake -GNinja \ + -DCMAKE_C_COMPILER=${CC} \ + -DCMAKE_CXX_COMPILER=${CXX} \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DLIBCXX_ABI_UNSTABLE=OFF \ + -DLLVM_USE_SANITIZER=${LIBCXX_SANITIZER} \ + -DLLVM_BUILD_32_BITS=${BUILD_32_BITS} \ + -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLIBCXX_INCLUDE_TESTS=OFF \ + -DLIBCXX_INCLUDE_BENCHMARKS=OFF \ + -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ + ../llvm-project/runtimes/ +cmake --build . -- cxx cxxabi +cd .. diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 1d0864b942..9b488295a3 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -4,20 +4,27 @@ on: push: {} pull_request: {} +env: + CMAKE_GENERATOR: Ninja + +permissions: + contents: read + jobs: - job: + build_and_test_default: name: bazel.${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-2022] - + os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: mount bazel cache - uses: actions/cache@v2.0.0 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 env: cache-name: bazel-cache with: diff --git a/.github/workflows/build-and-test-min-cmake.yml b/.github/workflows/build-and-test-min-cmake.yml new file mode 100644 index 0000000000..ef00f9d4c9 --- /dev/null +++ b/.github/workflows/build-and-test-min-cmake.yml @@ -0,0 +1,54 @@ +name: build-and-test-min-cmake + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +permissions: + contents: read + +env: + CMAKE_GENERATOR: Ninja + +jobs: + job: + name: ${{ matrix.os }}.min-cmake + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # latest + with: + cmakeVersion: 3.13.0 + + - name: create build environment + run: cmake -E make_directory $RUNNER_WORKSPACE/_build + + - name: setup cmake initial cache + run: touch compiler-cache.cmake + + - name: configure cmake + env: + CXX: ${{ matrix.compiler }} + shell: bash + working-directory: ${{ runner.workspace }}/_build + run: > + cmake -C ${{ github.workspace }}/compiler-cache.cmake + $GITHUB_WORKSPACE + -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON + -DCMAKE_CXX_VISIBILITY_PRESET=hidden + -DCMAKE_VISIBILITY_INLINES_HIDDEN=ON + + - name: build + shell: bash + working-directory: ${{ runner.workspace }}/_build + run: cmake --build . diff --git a/.github/workflows/build-and-test-perfcounters.yml b/.github/workflows/build-and-test-perfcounters.yml index bb5a43f375..829841e584 100644 --- a/.github/workflows/build-and-test-perfcounters.yml +++ b/.github/workflows/build-and-test-perfcounters.yml @@ -6,6 +6,12 @@ on: pull_request: branches: [ main ] +env: + CMAKE_GENERATOR: Ninja + +permissions: + contents: read + jobs: job: # TODO(dominic): Extend this to include compiler and set through env: CC/CXX. @@ -14,21 +20,20 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, ubuntu-20.04] + os: [ubuntu-latest] build_type: ['Release', 'Debug'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: install libpfm - run: sudo apt -y install libpfm4-dev - - - name: setup cmake - uses: jwlawson/actions-setup-cmake@v1.9 - with: - cmake-version: '3.5.1' + run: | + sudo apt update + sudo apt -y install libpfm4-dev - name: create build environment - run: cmake -E make_directory ${{ runner.workspace }}/_build + run: cmake -E make_directory $RUNNER_WORKSPACE/_build - name: configure cmake shell: bash @@ -52,46 +57,3 @@ jobs: # working-directory: ${{ runner.workspace }}/_build # run: ctest -C ${{ matrix.build_type }} --rerun-failed --output-on-failure - ubuntu-16_04: - name: ubuntu-16.04.${{ matrix.build_type }} - runs-on: [ubuntu-latest] - strategy: - fail-fast: false - matrix: - build_type: ['Release', 'Debug'] - container: ubuntu:16.04 - steps: - - uses: actions/checkout@v2 - - - name: install required bits - run: | - apt update - apt -y install clang cmake g++ git - - - name: install libpfm - run: apt -y install libpfm4-dev - - - name: create build environment - run: cmake -E make_directory $GITHUB_WORKSPACE/_build - - - name: configure cmake - shell: bash - working-directory: ${{ github.workspace }}/_build - run: > - cmake $GITHUB_WORKSPACE - -DBENCHMARK_ENABLE_LIBPFM=1 - -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON - -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} - - - name: build - shell: bash - working-directory: ${{ github.workspace }}/_build - run: cmake --build . --config ${{ matrix.build_type }} - - # Skip testing, for now. It seems perf_event_open does not succeed on the - # hosting machine, very likely a permissions issue. - # TODO(mtrofin): Enable test. - # - name: test - # shell: bash - # working-directory: ${{ runner.workspace }}/_build - # run: ctest -C ${{ matrix.build_type }} --rerun-failed --output-on-failure diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index d7406a7332..b965c17a7a 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -1,8 +1,16 @@ name: build-and-test on: - push: {} - pull_request: {} + push: + branches: [ main ] + pull_request: + branches: [ main ] + +permissions: + contents: read + +env: + CMAKE_GENERATOR: Ninja jobs: # TODO: add 32-bit builds (g++ and clang++) for ubuntu @@ -15,47 +23,32 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, ubuntu-20.04, macos-latest] + os: [ubuntu-24.04, ubuntu-22.04, ubuntu-24.04-arm, macos-latest] build_type: ['Release', 'Debug'] - compiler: [g++, clang++] + compiler: ['g++', 'clang++'] lib: ['shared', 'static'] steps: - - uses: actions/checkout@v2 - - - name: create build environment - run: cmake -E make_directory ${{ runner.workspace }}/_build - - - name: setup cmake initial cache - run: touch compiler-cache.cmake + - name: Install dependencies (macos) + if: runner.os == 'macOS' + run: brew install ninja - - name: setup lto - # Workaround for enabling -flto on old GCC versions - if: matrix.build_type == 'Release' && startsWith(matrix.compiler, 'g++') && matrix.os != 'macos-latest' - run: > - echo 'set (CMAKE_CXX_FLAGS -flto CACHE STRING "")' >> compiler-cache.cmake; - echo 'set (CMAKE_RANLIB /usr/bin/gcc-ranlib CACHE FILEPATH "")' >> compiler-cache.cmake; - echo 'set (CMAKE_AR /usr/bin/gcc-ar CACHE FILEPATH "")' >> compiler-cache.cmake; - echo 'set (CMAKE_NM /usr/bin/gcc-nm CACHE FILEPATH "")' >> compiler-cache.cmake; - - - name: configure cmake - env: - CXX: ${{ matrix.compiler }} - shell: bash - working-directory: ${{ runner.workspace }}/_build - run: > - cmake -C ${{ github.workspace }}/compiler-cache.cmake - $GITHUB_WORKSPACE - -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON - -DBUILD_SHARED_LIBS=${{ matrix.lib == 'shared' }} - -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} - -DCMAKE_CXX_VISIBILITY_PRESET=hidden - -DCMAKE_VISIBILITY_INLINES_HIDDEN=ON + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: build - shell: bash - working-directory: ${{ runner.workspace }}/_build - run: cmake --build . --config ${{ matrix.build_type }} + uses: threeal/cmake-action@725d1314ccf9ea922805d7e3f9d9bcbca892b406 # v2.1.0 + with: + build-dir: ${{ runner.workspace }}/_build + cxx-compiler: ${{ matrix.compiler }} + options: | + BENCHMARK_DOWNLOAD_DEPENDENCIES=ON + BUILD_SHARED_LIBS=${{ matrix.lib == 'shared' }} + CMAKE_BUILD_TYPE=${{ matrix.build_type }} + CMAKE_CXX_COMPILER=${{ matrix.compiler }} + CMAKE_CXX_VISIBILITY_PRESET=hidden + CMAKE_VISIBILITY_INLINES_HIDDEN=ON - name: test shell: bash @@ -72,10 +65,8 @@ jobs: fail-fast: false matrix: msvc: - - VS-16-2019 + - VS-18-2025 - VS-17-2022 - arch: - - x64 build_type: - Debug - Release @@ -83,170 +74,93 @@ jobs: - shared - static include: - - msvc: VS-16-2019 - os: windows-2019 - generator: 'Visual Studio 16 2019' + - msvc: VS-18-2025 + os: windows-2025 + generator: 'Visual Studio 18 2026' - msvc: VS-17-2022 os: windows-2022 generator: 'Visual Studio 17 2022' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # latest - name: configure cmake run: > - cmake -S . -B _build/ - -A ${{ matrix.arch }} + cmake -S . -B $env:RUNNER_WORKSPACE/_build/ -G "${{ matrix.generator }}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBUILD_SHARED_LIBS=${{ matrix.lib == 'shared' }} - - - name: build - run: cmake --build _build/ --config ${{ matrix.build_type }} - - - name: setup test environment - # Make sure gmock and benchmark DLLs can be found - run: > - echo "$((Get-Item .).FullName)/_build/bin/${{ matrix.build_type }}" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append; - echo "$((Get-Item .).FullName)/_build/src/${{ matrix.build_type }}" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append; - - - name: test - run: ctest --test-dir _build/ -C ${{ matrix.build_type }} -VV - - ubuntu-16_04: - name: ubuntu-16.04.${{ matrix.build_type }}.${{ matrix.compiler }} - runs-on: [ubuntu-latest] - strategy: - fail-fast: false - matrix: - build_type: ['Release', 'Debug'] - compiler: [g++, clang++] - container: ubuntu:16.04 - steps: - - uses: actions/checkout@v2 - - - name: install required bits - run: | - apt update - apt -y install clang cmake g++ git - - - name: create build environment - run: cmake -E make_directory $GITHUB_WORKSPACE/_build - - - name: setup cmake initial cache - run: touch compiler-cache.cmake - - - name: setup lto - # Workaround for enabling -flto on old GCC versions - # -Wl,--no-as-needed is needed to avoid the following linker error: - # - # /usr/lib/gcc/x86_64-linux-gnu/5/libstdc++.so: undefined reference to `pthread_create' - # - if: matrix.build_type == 'Release' && startsWith(matrix.compiler, 'g++') - run: > - echo 'set (CMAKE_CXX_FLAGS "-Wl,--no-as-needed -flto" CACHE STRING "")' >> compiler-cache.cmake; - echo 'set (CMAKE_RANLIB "/usr/bin/gcc-ranlib" CACHE FILEPATH "")' >> compiler-cache.cmake; - echo 'set (CMAKE_AR "/usr/bin/gcc-ar" CACHE FILEPATH "")' >> compiler-cache.cmake; - echo 'set (CMAKE_NM "/usr/bin/gcc-nm" CACHE FILEPATH "")' >> compiler-cache.cmake; - - - name: configure cmake env: - CXX: ${{ matrix.compiler }} - shell: bash - working-directory: ${{ github.workspace }}/_build - run: > - cmake -C ../compiler-cache.cmake .. - -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON - -DBUILD_SHARED_LIBS=${{ matrix.lib == 'shared' }} - -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} - -DCMAKE_CXX_VISIBILITY_PRESET=hidden - -DCMAKE_VISIBILITY_INLINES_HIDDEN=ON + RUNNER_WORKSPACE: ${{ runner.workspace }} - name: build - shell: bash - working-directory: ${{ github.workspace }}/_build - run: cmake --build . --config ${{ matrix.build_type }} + run: cmake --build $env:RUNNER_WORKSPACE/_build/ --config ${{ matrix.build_type }} + env: + RUNNER_WORKSPACE: ${{ runner.workspace }} - name: test - shell: bash - working-directory: ${{ github.workspace }}/_build - run: ctest -C ${{ matrix.build_type }} -VV + run: ctest --test-dir $env:RUNNER_WORKSPACE/_build/ -C ${{ matrix.build_type }} -VV + env: + RUNNER_WORKSPACE: ${{ runner.workspace }} - ubuntu-14_04: - name: ubuntu-14.04.${{ matrix.build_type }}.${{ matrix.compiler }} - runs-on: [ubuntu-latest] + msys2: + name: ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} + runs-on: ${{ matrix.os }} + defaults: + run: + shell: msys2 {0} strategy: fail-fast: false matrix: - build_type: ['Release', 'Debug'] - compiler: [g++-4.8, clang++-3.6] - include: - - compiler: g++-6 - build_type: 'Debug' - run_tests: true - - compiler: g++-6 - build_type: 'Release' - run_tests: true - container: ubuntu:14.04 - steps: - - uses: actions/checkout@v2 - - - name: install required bits - run: | - sudo apt update - sudo apt -y install clang-3.6 cmake3 g++-4.8 git - - - name: install other bits - if: ${{ matrix.compiler }} == g++-6 - run: | - sudo apt -y install software-properties-common - sudo add-apt-repository -y "ppa:ubuntu-toolchain-r/test" - sudo apt update - sudo apt -y install g++-6 - - - name: create build environment - run: cmake -E make_directory $GITHUB_WORKSPACE/_build - - - name: setup cmake initial cache - run: touch compiler-cache.cmake - - - name: setup lto - # Workaround for enabling -flto on old GCC versions - # -Wl,--no-as-needed is needed to avoid the following linker error: - # - # /usr/lib/gcc/x86_64-linux-gnu/6/libstdc++.so: undefined reference to `pthread_create' - # - if: matrix.build_type == 'Release' && startsWith(matrix.compiler, 'g++') - run: > - COMPILER=${{ matrix.compiler }}; - VERSION=${COMPILER#g++-}; - PREFIX=/usr/bin/gcc; - echo "set (CMAKE_CXX_FLAGS \"-Wl,--no-as-needed -flto\" CACHE STRING \"\")" >> compiler-cache.cmake; - echo "set (CMAKE_RANLIB \"$PREFIX-ranlib-$VERSION\" CACHE FILEPATH \"\")" >> compiler-cache.cmake; - echo "set (CMAKE_AR \"$PREFIX-ar-$VERSION\" CACHE FILEPATH \"\")" >> compiler-cache.cmake; - echo "set (CMAKE_NM \"$PREFIX-nm-$VERSION\" CACHE FILEPAT \"\")" >> compiler-cache.cmake; + os: [ windows-latest ] + msys2: + - { msystem: MINGW64, arch: x86_64, family: GNU, compiler: g++ } + - { msystem: CLANG64, arch: x86_64, family: LLVM, compiler: clang++ } + - { msystem: UCRT64, arch: x86_64, family: GNU, compiler: g++ } + build_type: + - Debug + - Release + lib: + - shared + - static + steps: + - name: setup msys2 + uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2.32.0 + with: + cache: false + msystem: ${{ matrix.msys2.msystem }} + update: true + install: >- + git + base-devel + pacboy: >- + gcc:p + clang:p + cmake:p + ninja:p + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # NOTE: we can't use cmake actions here as we need to do everything in msys2 shell. - name: configure cmake env: - CXX: ${{ matrix.compiler }} - shell: bash - working-directory: ${{ github.workspace }}/_build + CXX: ${{ matrix.msys2.compiler }} run: > - cmake -C ../compiler-cache.cmake .. - -DBENCHMARK_DOWNLOAD_DEPENDENCIES=${{ matrix.run_tests }} - -DBENCHMARK_ENABLE_TESTING=${{ matrix.run_tests }} + cmake -S . -B _build/ + -GNinja + -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBUILD_SHARED_LIBS=${{ matrix.lib == 'shared' }} - -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} - -DCMAKE_CXX_VISIBILITY_PRESET=hidden - -DCMAKE_VISIBILITY_INLINES_HIDDEN=ON - name: build - shell: bash - working-directory: ${{ github.workspace }}/_build - run: cmake --build . --config ${{ matrix.build_type }} + run: cmake --build _build/ --config ${{ matrix.build_type }} - name: test - if: ${{ matrix.run_tests }} - shell: bash - working-directory: ${{ github.workspace }}/_build + working-directory: _build run: ctest -C ${{ matrix.build_type }} -VV diff --git a/.github/workflows/clang-format-lint.yml b/.github/workflows/clang-format-lint.yml index 75775c7cfc..e7c2c9d40d 100644 --- a/.github/workflows/clang-format-lint.yml +++ b/.github/workflows/clang-format-lint.yml @@ -3,15 +3,22 @@ on: push: {} pull_request: {} +env: + CMAKE_GENERATOR: Ninja + +permissions: + contents: read + jobs: - build: + job: + name: check-clang-format runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: DoozyX/clang-format-lint-action@v0.13 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: DoozyX/clang-format-lint-action@bcb4eb2cb0d707ee4f3e5cc3b456eb075f12cf73 # v0.20 with: - source: './include/benchmark ./src ./test' - extensions: 'h,cc' - clangFormatVersion: 12 - style: Google + source: './include/benchmark ./src ./test ./bindings' + clangFormatVersion: 18 diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy-lint.yml similarity index 61% rename from .github/workflows/clang-tidy.yml rename to .github/workflows/clang-tidy-lint.yml index 978171df0d..42e65cbf72 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy-lint.yml @@ -4,6 +4,12 @@ on: push: {} pull_request: {} +env: + CMAKE_GENERATOR: Ninja + +permissions: + contents: read + jobs: job: name: run-clang-tidy @@ -11,17 +17,19 @@ jobs: strategy: fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: install clang-tidy run: sudo apt update && sudo apt -y install clang-tidy - name: create build environment - run: cmake -E make_directory ${{ runner.workspace }}/_build + run: cmake -E make_directory ${{ github.workspace }}/_build - name: configure cmake shell: bash - working-directory: ${{ runner.workspace }}/_build + working-directory: ${{ github.workspace }}/_build run: > cmake $GITHUB_WORKSPACE -DBENCHMARK_ENABLE_ASSEMBLY_TESTS=OFF @@ -34,5 +42,5 @@ jobs: - name: run shell: bash - working-directory: ${{ runner.workspace }}/_build - run: run-clang-tidy + working-directory: ${{ github.workspace }}/_build + run: run-clang-tidy -config-file=$GITHUB_WORKSPACE/.clang-tidy diff --git a/.github/workflows/doxygen.yml b/.github/workflows/doxygen.yml index dc55011b8b..5d8246cb51 100644 --- a/.github/workflows/doxygen.yml +++ b/.github/workflows/doxygen.yml @@ -6,20 +6,30 @@ on: pull_request: branches: [main] +env: + CMAKE_GENERATOR: Ninja + +permissions: + contents: read + jobs: build-and-deploy: name: Build HTML documentation runs-on: ubuntu-latest steps: - name: Fetching sources - uses: actions/checkout@v2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Installing build dependencies run: | sudo apt update - sudo apt install cmake doxygen gcc git + sudo apt install doxygen gcc git + - name: Creating build directory - run: | - mkdir build + run: mkdir build + - name: Building HTML documentation with Doxygen run: | cmake -S . -B build -DBENCHMARK_ENABLE_TESTING:BOOL=OFF -DBENCHMARK_ENABLE_DOXYGEN:BOOL=ON -DBENCHMARK_INSTALL_DOCS:BOOL=ON diff --git a/.github/workflows/ossf.yml b/.github/workflows/ossf.yml new file mode 100644 index 0000000000..8388c5784d --- /dev/null +++ b/.github/workflows/ossf.yml @@ -0,0 +1,29 @@ +name: OSSF Scorecard Weekly + +on: + schedule: + - cron: '0 0 * * 0' # Runs every Sunday at midnight UTC + workflow_dispatch: + +permissions: + contents: read + +jobs: + ossf-scorecard: + # To write a badge + permissions: + id-token: write + + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Run analysis + uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 + with: + publish_results: true + results_file: ossf_scorecard.json + results_format: json diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000000..7cbab26fc8 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,25 @@ +name: python + Bazel pre-commit checks + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +permissions: + contents: read + +jobs: + pre-commit: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + python-version: 3.12 + - name: Run pre-commit checks + run: uv run --only-group=dev pre-commit run --all-files --verbose --show-diff-on-failure diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml deleted file mode 100644 index 0f73a58232..0000000000 --- a/.github/workflows/pylint.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: pylint - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - pylint: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pylint pylint-exit conan - - name: Run pylint - run: | - pylint `find . -name '*.py'|xargs` || pylint-exit $? diff --git a/.github/workflows/sanitizer.yml b/.github/workflows/sanitizer.yml index bbfc782200..b106c45d76 100644 --- a/.github/workflows/sanitizer.yml +++ b/.github/workflows/sanitizer.yml @@ -4,22 +4,27 @@ on: push: {} pull_request: {} +permissions: + contents: read + env: + CMAKE_GENERATOR: Ninja UBSAN_OPTIONS: "print_stacktrace=1" jobs: job: - name: ${{ matrix.sanitizer }}.${{ matrix.build_type }}.${{ matrix.compiler }} + name: ${{ matrix.sanitizer }}.${{ matrix.build_type }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: build_type: ['Debug', 'RelWithDebInfo'] - sanitizer: ['asan', 'ubsan', 'tsan'] - compiler: ['clang', 'gcc'] - # TODO: add 'msan' above. currently failing and needs investigation. + sanitizer: ['asan', 'ubsan', 'tsan', 'msan'] + steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: configure msan env if: matrix.sanitizer == 'msan' @@ -45,40 +50,47 @@ jobs: echo "EXTRA_FLAGS=-g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" >> $GITHUB_ENV echo "LIBCXX_SANITIZER=Thread" >> $GITHUB_ENV - - name: configure clang - if: matrix.compiler == 'clang' + - name: fine-tune asan options + # in asan we get an error from std::regex. ignore it. + if: matrix.sanitizer == 'asan' run: | - echo "CC=clang" >> $GITHUB_ENV - echo "CXX=clang++" >> $GITHUB_ENV + echo "ASAN_OPTIONS=alloc_dealloc_mismatch=0" >> $GITHUB_ENV + + - name: setup clang + uses: egor-tensin/setup-clang@23bc15cd4207e45f1566447deb48f4ff3ef932cb # v2.3 + with: + version: latest + platform: x64 - - name: configure gcc - if: matrix.compiler == 'gcc' + - name: configure clang run: | - sudo apt update && sudo apt -y install gcc-10 g++-10 - echo "CC=gcc-10" >> $GITHUB_ENV - echo "CXX=g++-10" >> $GITHUB_ENV + echo "CC=cc" >> $GITHUB_ENV + echo "CXX=c++" >> $GITHUB_ENV - - name: install llvm stuff - if: matrix.compiler == 'clang' + - name: build libc++ (non-asan) + if: matrix.sanitizer != 'asan' run: | - "${GITHUB_WORKSPACE}/.github/.libcxx-setup.sh" - echo "EXTRA_CXX_FLAGS=\"-stdlib=libc++\"" >> $GITHUB_ENV + "${GITHUB_WORKSPACE}/.github/libcxx-setup.sh" + echo "EXTRA_CXX_FLAGS=-stdlib=libc++ -L${GITHUB_WORKSPACE}/llvm-build/lib -lc++abi -I${GITHUB_WORKSPACE}/llvm-build/include/c++/v1 -Isystem${GITHUB_WORKSPACE}/llvm-build/include/c++/v1 -Wl,-rpath,${GITHUB_WORKSPACE}/llvm-build/lib" >> $GITHUB_ENV - name: create build environment - run: cmake -E make_directory ${{ runner.workspace }}/_build + run: cmake -E make_directory ${RUNNER_WORKSPACE}/_build + env: + RUNNER_WORKSPACE: ${{ runner.workspace }} - name: configure cmake shell: bash working-directory: ${{ runner.workspace }}/_build run: > - cmake $GITHUB_WORKSPACE + VERBOSE=1 + cmake -GNinja $GITHUB_WORKSPACE -DBENCHMARK_ENABLE_ASSEMBLY_TESTS=OFF -DBENCHMARK_ENABLE_LIBPFM=OFF -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON - -DCMAKE_C_COMPILER=${{ env.CC }} - -DCMAKE_CXX_COMPILER=${{ env.CXX }} - -DCMAKE_C_FLAGS="${{ env.EXTRA_FLAGS }}" - -DCMAKE_CXX_FLAGS="${{ env.EXTRA_FLAGS }} ${{ env.EXTRA_CXX_FLAGS }}" + -DCMAKE_C_COMPILER=${CC} + -DCMAKE_CXX_COMPILER=${CXX} + -DCMAKE_C_FLAGS="${EXTRA_FLAGS}" + -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS} ${EXTRA_CXX_FLAGS}" -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} - name: build diff --git a/.github/workflows/test_bindings.yml b/.github/workflows/test_bindings.yml index 4a580ebe04..1899721b2d 100644 --- a/.github/workflows/test_bindings.yml +++ b/.github/workflows/test_bindings.yml @@ -6,19 +6,60 @@ on: pull_request: branches: [main] +env: + CMAKE_GENERATOR: Ninja + +permissions: + contents: read + jobs: python_bindings: - runs-on: ubuntu-latest + name: Test Python ${{ matrix.python-version }} bindings on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, macos-latest, windows-latest ] + python-version: [ "3.10", "3.11", "3.12", "3.13" ] steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install Python bindings on ${{ matrix.os }} + run: python -m pip install . + - name: Run example on ${{ matrix.os }} under Python ${{ matrix.python-version }} + run: python bindings/python/google_benchmark/example.py + + rust_bindings: + name: Test Rust bindings on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, macos-latest, windows-latest ] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 with: - python-version: 3.8 - - name: Install benchmark - run: - python setup.py install - - name: Run example bindings - run: - python bindings/python/google_benchmark/example.py + toolchain: stable + - name: Install Ninja (macOS) + if: runner.os == 'macOS' + run: brew install ninja + - name: Run Rust tests natively via Cargo + run: cargo test + working-directory: bindings/rust + - name: Run Rust tests via CMake + run: | + cmake -S . -B build -DBENCHMARK_ENABLE_RUST_BINDINGS=ON -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON + cmake --build build + cd build && ctest -R rust_bindings_tests diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 6c2256970b..4b20127d02 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -6,26 +6,32 @@ on: types: - published +env: + CMAKE_GENERATOR: Ninja + +permissions: + contents: read + jobs: build_sdist: name: Build source distribution runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v3 - - - name: Install Python 3.9 - uses: actions/setup-python@v3 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - python-version: 3.9 - - - name: Build and check sdist - run: | - python setup.py sdist - - name: Upload sdist - uses: actions/upload-artifact@v3 + fetch-depth: 0 + persist-credentials: false + - name: Install Python 3.12 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - run: python -m pip install build + - name: Build sdist + run: python -m build --sdist + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: dist + name: dist-sdist path: dist/*.tar.gz build_wheels: @@ -33,51 +39,53 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - + os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-latest, windows-latest] steps: - name: Check out Google Benchmark - uses: actions/checkout@v3 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false - - name: Set up Python 3.9 - uses: actions/setup-python@v3 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + name: Install Python 3.12 + with: + python-version: "3.12" + - name: Install the latest version of uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: - python-version: 3.9 + enable-cache: false - - name: Install and run cibuildwheel on ${{ matrix.os }} + - name: Build wheels on ${{ matrix.os }} using cibuildwheel + uses: pypa/cibuildwheel@294735312765b09d24a2fbec22660ce817587d55 # v4.1.0 env: - CIBW_BUILD: 'cp37-* cp38-* cp39-* cp310-*' - CIBW_SKIP: "cp37-*-arm64 *-musllinux_*" - # TODO: Build ppc64le, aarch64 using some other trick - CIBW_ARCHS_LINUX: x86_64 - CIBW_ARCHS_MACOS: x86_64 arm64 - CIBW_ARCHS_WINDOWS: AMD64 - CIBW_BEFORE_ALL_LINUX: > - curl -O --retry-delay 5 --retry 5 https://copr.fedorainfracloud.org/coprs/vbatts/bazel/repo/epel-7/vbatts-bazel-epel-7.repo && - cp vbatts-bazel-epel-7.repo /etc/yum.repos.d/bazel.repo && - yum install -y bazel4 + CIBW_BUILD: "cp310-* cp311-* cp312-*" + CIBW_BUILD_FRONTEND: "build[uv]" + CIBW_SKIP: "*-musllinux_*" + CIBW_ARCHS: auto64 + CIBW_BEFORE_ALL_LINUX: bash .github/install_bazel.sh + # Grab the rootless Bazel installation inside the container. + CIBW_ENVIRONMENT_LINUX: PATH=$PATH:$HOME/bin CIBW_TEST_COMMAND: python {project}/bindings/python/google_benchmark/example.py - run: | - pip install cibuildwheel - python -m cibuildwheel --output-dir wheelhouse + # unused by Bazel, but needed explicitly by delocate on MacOS. + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.os == 'macos-15-intel' && 10.14 || 11.0 }} - name: Upload Google Benchmark ${{ matrix.os }} wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: dist - path: ./wheelhouse/*.whl + name: dist-${{ matrix.os }} + path: wheelhouse/*.whl pypi_upload: name: Publish google-benchmark wheels to PyPI needs: [build_sdist, build_wheels] runs-on: ubuntu-latest + permissions: + id-token: write steps: - - uses: actions/download-artifact@v3 - with: - name: dist - path: dist - - - uses: pypa/gh-action-pypi-publish@v1.5.0 - with: - user: __token__ - password: ${{ secrets.PYPI_PASSWORD }} + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: dist + pattern: dist-* + merge-multiple: true + - uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # release/v1 diff --git a/.gitignore b/.gitignore index 704f56c257..53704eba79 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ *.exe *.dylib *.cmake -!/cmake/*.cmake +!/cmake/**/*.cmake !/test/AssemblyTests.cmake *~ *.swp @@ -46,6 +46,7 @@ rules.ninja # bazel output symlinks. bazel-* +MODULE.bazel.lock # out-of-source build top-level folders. build/ @@ -65,3 +66,7 @@ CMakeSettings.json # Python build stuff dist/ *.egg-info* +uv.lock + +# Rust build stuff +/bindings/rust/target/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..57af012f7a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +repos: + - repo: https://github.com/keith/pre-commit-buildifier + rev: 8.2.1 + hooks: + - id: buildifier + - id: buildifier-lint + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.18.2 + hooks: + - id: mypy + types_or: [ python, pyi ] + args: [ "--ignore-missing-imports", "--scripts-are-modules" ] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.14.0 + hooks: + - id: ruff-check + args: [ --fix, --exit-non-zero-on-fix ] + - id: ruff-format diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 8cfed3d10d..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,208 +0,0 @@ -sudo: required -dist: trusty -language: cpp - -matrix: - include: - - compiler: gcc - addons: - apt: - packages: - - lcov - env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Coverage - - compiler: gcc - addons: - apt: - packages: - - g++-multilib - - libc6:i386 - env: - - COMPILER=g++ - - C_COMPILER=gcc - - BUILD_TYPE=Debug - - BUILD_32_BITS=ON - - EXTRA_FLAGS="-m32" - - compiler: gcc - addons: - apt: - packages: - - g++-multilib - - libc6:i386 - env: - - COMPILER=g++ - - C_COMPILER=gcc - - BUILD_TYPE=Release - - BUILD_32_BITS=ON - - EXTRA_FLAGS="-m32" - - compiler: gcc - env: - - INSTALL_GCC6_FROM_PPA=1 - - COMPILER=g++-6 C_COMPILER=gcc-6 BUILD_TYPE=Debug - - ENABLE_SANITIZER=1 - - EXTRA_FLAGS="-fno-omit-frame-pointer -g -O2 -fsanitize=undefined,address -fuse-ld=gold" - # Clang w/ libc++ - - compiler: clang - dist: xenial - addons: - apt: - packages: - clang-3.8 - env: - - INSTALL_GCC6_FROM_PPA=1 - - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - - LIBCXX_BUILD=1 - - EXTRA_CXX_FLAGS="-stdlib=libc++" - - compiler: clang - dist: xenial - addons: - apt: - packages: - clang-3.8 - env: - - INSTALL_GCC6_FROM_PPA=1 - - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release - - LIBCXX_BUILD=1 - - EXTRA_CXX_FLAGS="-stdlib=libc++" - # Clang w/ 32bit libc++ - - compiler: clang - dist: xenial - addons: - apt: - packages: - - clang-3.8 - - g++-multilib - - libc6:i386 - env: - - INSTALL_GCC6_FROM_PPA=1 - - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - - LIBCXX_BUILD=1 - - BUILD_32_BITS=ON - - EXTRA_FLAGS="-m32" - - EXTRA_CXX_FLAGS="-stdlib=libc++" - # Clang w/ 32bit libc++ - - compiler: clang - dist: xenial - addons: - apt: - packages: - - clang-3.8 - - g++-multilib - - libc6:i386 - env: - - INSTALL_GCC6_FROM_PPA=1 - - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release - - LIBCXX_BUILD=1 - - BUILD_32_BITS=ON - - EXTRA_FLAGS="-m32" - - EXTRA_CXX_FLAGS="-stdlib=libc++" - # Clang w/ libc++, ASAN, UBSAN - - compiler: clang - dist: xenial - addons: - apt: - packages: - clang-3.8 - env: - - INSTALL_GCC6_FROM_PPA=1 - - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - - LIBCXX_BUILD=1 LIBCXX_SANITIZER="Undefined;Address" - - ENABLE_SANITIZER=1 - - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=undefined,address -fno-sanitize-recover=all" - - EXTRA_CXX_FLAGS="-stdlib=libc++" - - UBSAN_OPTIONS=print_stacktrace=1 - # Clang w/ libc++ and MSAN - - compiler: clang - dist: xenial - addons: - apt: - packages: - clang-3.8 - env: - - INSTALL_GCC6_FROM_PPA=1 - - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - - LIBCXX_BUILD=1 LIBCXX_SANITIZER=MemoryWithOrigins - - ENABLE_SANITIZER=1 - - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins" - - EXTRA_CXX_FLAGS="-stdlib=libc++" - # Clang w/ libc++ and MSAN - - compiler: clang - dist: xenial - addons: - apt: - packages: - clang-3.8 - env: - - INSTALL_GCC6_FROM_PPA=1 - - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=RelWithDebInfo - - LIBCXX_BUILD=1 LIBCXX_SANITIZER=Thread - - ENABLE_SANITIZER=1 - - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" - - EXTRA_CXX_FLAGS="-stdlib=libc++" - - os: osx - osx_image: xcode8.3 - compiler: clang - env: - - COMPILER=clang++ - - BUILD_TYPE=Release - - BUILD_32_BITS=ON - - EXTRA_FLAGS="-m32" - -before_script: - - if [ -n "${LIBCXX_BUILD}" ]; then - source .libcxx-setup.sh; - fi - - if [ -n "${ENABLE_SANITIZER}" ]; then - export EXTRA_OPTIONS="-DBENCHMARK_ENABLE_ASSEMBLY_TESTS=OFF"; - else - export EXTRA_OPTIONS=""; - fi - - mkdir -p build && cd build - -before_install: - - if [ -z "$BUILD_32_BITS" ]; then - export BUILD_32_BITS=OFF && echo disabling 32 bit build; - fi - - if [ -n "${INSTALL_GCC6_FROM_PPA}" ]; then - sudo add-apt-repository -y "ppa:ubuntu-toolchain-r/test"; - sudo apt-get update --option Acquire::Retries=100 --option Acquire::http::Timeout="60"; - fi - -install: - - if [ -n "${INSTALL_GCC6_FROM_PPA}" ]; then - travis_wait sudo -E apt-get -yq --no-install-suggests --no-install-recommends install g++-6; - fi - - if [ "${TRAVIS_OS_NAME}" == "linux" -a "${BUILD_32_BITS}" == "OFF" ]; then - travis_wait sudo -E apt-get -y --no-install-suggests --no-install-recommends install llvm-3.9-tools; - sudo cp /usr/lib/llvm-3.9/bin/FileCheck /usr/local/bin/; - fi - - if [ "${BUILD_TYPE}" == "Coverage" -a "${TRAVIS_OS_NAME}" == "linux" ]; then - PATH=~/.local/bin:${PATH}; - pip install --user --upgrade pip; - travis_wait pip install --user cpp-coveralls; - fi - - if [ "${C_COMPILER}" == "gcc-7" -a "${TRAVIS_OS_NAME}" == "osx" ]; then - rm -f /usr/local/include/c++; - brew update; - travis_wait brew install gcc@7; - fi - - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then - sudo apt-get update -qq; - sudo apt-get install -qq unzip cmake3; - wget https://github.com/bazelbuild/bazel/releases/download/3.2.0/bazel-3.2.0-installer-linux-x86_64.sh --output-document bazel-installer.sh; - travis_wait sudo bash bazel-installer.sh; - fi - - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then - curl -L -o bazel-installer.sh https://github.com/bazelbuild/bazel/releases/download/3.2.0/bazel-3.2.0-installer-darwin-x86_64.sh; - travis_wait sudo bash bazel-installer.sh; - fi - -script: - - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_C_FLAGS="${EXTRA_FLAGS}" -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS} ${EXTRA_CXX_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} ${EXTRA_OPTIONS} .. - - make - - ctest -C ${BUILD_TYPE} --output-on-failure - - bazel test -c dbg --define google_benchmark.have_regex=posix --announce_rc --verbose_failures --test_output=errors --keep_going //test/... - -after_success: - - if [ "${BUILD_TYPE}" == "Coverage" -a "${TRAVIS_OS_NAME}" == "linux" ]; then - coveralls --include src --include include --gcov-options '\-lp' --root .. --build-root .; - fi diff --git a/.ycm_extra_conf.py b/.ycm_extra_conf.py index 5649ddcc74..ffef1b4daf 100644 --- a/.ycm_extra_conf.py +++ b/.ycm_extra_conf.py @@ -1,25 +1,30 @@ import os + import ycm_core # These are the compilation flags that will be used in case there's no # compilation database set (by default, one is not set). # CHANGE THIS LIST OF FLAGS. YES, THIS IS THE DROID YOU HAVE BEEN LOOKING FOR. flags = [ -'-Wall', -'-Werror', -'-pedantic-errors', -'-std=c++0x', -'-fno-strict-aliasing', -'-O3', -'-DNDEBUG', -# ...and the same thing goes for the magic -x option which specifies the -# language that the files to be compiled are written in. This is mostly -# relevant for c++ headers. -# For a C project, you would set this to 'c' instead of 'c++'. -'-x', 'c++', -'-I', 'include', -'-isystem', '/usr/include', -'-isystem', '/usr/local/include', + "-Wall", + "-Werror", + "-pedantic-errors", + "-std=c++0x", + "-fno-strict-aliasing", + "-O3", + "-DNDEBUG", + # ...and the same thing goes for the magic -x option which specifies the + # language that the files to be compiled are written in. This is mostly + # relevant for c++ headers. + # For a C project, you would set this to 'c' instead of 'c++'. + "-x", + "c++", + "-I", + "include", + "-isystem", + "/usr/include", + "-isystem", + "/usr/local/include", ] @@ -29,87 +34,87 @@ # # Most projects will NOT need to set this to anything; you can just change the # 'flags' list of compilation flags. Notice that YCM itself uses that approach. -compilation_database_folder = '' +compilation_database_folder = "" -if os.path.exists( compilation_database_folder ): - database = ycm_core.CompilationDatabase( compilation_database_folder ) +if os.path.exists(compilation_database_folder): + database = ycm_core.CompilationDatabase(compilation_database_folder) else: - database = None + database = None + +SOURCE_EXTENSIONS = [".cc"] -SOURCE_EXTENSIONS = [ '.cc' ] def DirectoryOfThisScript(): - return os.path.dirname( os.path.abspath( __file__ ) ) - - -def MakeRelativePathsInFlagsAbsolute( flags, working_directory ): - if not working_directory: - return list( flags ) - new_flags = [] - make_next_absolute = False - path_flags = [ '-isystem', '-I', '-iquote', '--sysroot=' ] - for flag in flags: - new_flag = flag - - if make_next_absolute: - make_next_absolute = False - if not flag.startswith( '/' ): - new_flag = os.path.join( working_directory, flag ) - - for path_flag in path_flags: - if flag == path_flag: - make_next_absolute = True - break - - if flag.startswith( path_flag ): - path = flag[ len( path_flag ): ] - new_flag = path_flag + os.path.join( working_directory, path ) - break - - if new_flag: - new_flags.append( new_flag ) - return new_flags - - -def IsHeaderFile( filename ): - extension = os.path.splitext( filename )[ 1 ] - return extension in [ '.h', '.hxx', '.hpp', '.hh' ] - - -def GetCompilationInfoForFile( filename ): - # The compilation_commands.json file generated by CMake does not have entries - # for header files. So we do our best by asking the db for flags for a - # corresponding source file, if any. If one exists, the flags for that file - # should be good enough. - if IsHeaderFile( filename ): - basename = os.path.splitext( filename )[ 0 ] - for extension in SOURCE_EXTENSIONS: - replacement_file = basename + extension - if os.path.exists( replacement_file ): - compilation_info = database.GetCompilationInfoForFile( - replacement_file ) - if compilation_info.compiler_flags_: - return compilation_info - return None - return database.GetCompilationInfoForFile( filename ) - - -def FlagsForFile( filename, **kwargs ): - if database: - # Bear in mind that compilation_info.compiler_flags_ does NOT return a - # python list, but a "list-like" StringVec object - compilation_info = GetCompilationInfoForFile( filename ) - if not compilation_info: - return None - - final_flags = MakeRelativePathsInFlagsAbsolute( - compilation_info.compiler_flags_, - compilation_info.compiler_working_dir_ ) - else: - relative_to = DirectoryOfThisScript() - final_flags = MakeRelativePathsInFlagsAbsolute( flags, relative_to ) - - return { - 'flags': final_flags, - 'do_cache': True - } + return os.path.dirname(os.path.abspath(__file__)) + + +def MakeRelativePathsInFlagsAbsolute(flags, working_directory): + if not working_directory: + return list(flags) + new_flags = [] + make_next_absolute = False + path_flags = ["-isystem", "-I", "-iquote", "--sysroot="] + for flag in flags: + new_flag = flag + + if make_next_absolute: + make_next_absolute = False + if not flag.startswith("/"): + new_flag = os.path.join(working_directory, flag) + + for path_flag in path_flags: + if flag == path_flag: + make_next_absolute = True + break + + if flag.startswith(path_flag): + path = flag[len(path_flag) :] + new_flag = path_flag + os.path.join(working_directory, path) + break + + if new_flag: + new_flags.append(new_flag) + return new_flags + + +def IsHeaderFile(filename): + extension = os.path.splitext(filename)[1] + return extension in [".h", ".hxx", ".hpp", ".hh"] + + +def GetCompilationInfoForFile(filename): + # The compilation_commands.json file generated by CMake does not have + # entries for header files. So we do our best by asking the db for flags for + # a corresponding source file, if any. If one exists, the flags for that + # file should be good enough. + if IsHeaderFile(filename): + basename = os.path.splitext(filename)[0] + for extension in SOURCE_EXTENSIONS: + replacement_file = basename + extension + if os.path.exists(replacement_file): + compilation_info = database.GetCompilationInfoForFile( + replacement_file + ) + if compilation_info.compiler_flags_: + return compilation_info + return None + return database.GetCompilationInfoForFile(filename) + + +def FlagsForFile(filename, **kwargs): + if database: + # Bear in mind that compilation_info.compiler_flags_ does NOT return a + # python list, but a "list-like" StringVec object + compilation_info = GetCompilationInfoForFile(filename) + if not compilation_info: + return None + + final_flags = MakeRelativePathsInFlagsAbsolute( + compilation_info.compiler_flags_, + compilation_info.compiler_working_dir_, + ) + else: + relative_to = DirectoryOfThisScript() + final_flags = MakeRelativePathsInFlagsAbsolute(flags, relative_to) + + return {"flags": final_flags, "do_cache": True} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..72f2abe0f0 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,44 @@ +# AI usage + +> [!NOTE] +> The modern (as of 2026-06) AI is a misnomer, +> - it is not conscious / not a consciousness, +> - it is not sentient, +> - it is not intelligent, +> - it does not think, +> - it does not understand the code, +> - it is merely a next-token guesser, +> +> ... therefore it is merely an (hyper-) advanced IDE. + +> [!NOTE] +> A contribution is any externally-observable interaction with a project. + +> [!CAUTION] +> Failure to follow the following rules *may* result in repercussions, +> possibly without a prior warning. + +Rules: +1. It is acceptable to use AI when producing contributions. +> [!WARNING] +> Any and all AI usage **MUST** be fully and explicitly disclosed +> in **every** contribution. +2. All contributions shall be done by conscious, sentient beings. + Fully autonomous contributions by bots are prohibited[^1]. +> [!WARNING] +> Dear contributor, the AI is *your* *tool*, and its output is for *your* +> *consumption*. It is **your** responsibility to consume said output, +> interpret it, and then produce the contribution itself. +> **DO NOT** just query it and post the output, +> *especially* so for all non-code contributions! +3. The contributor (conscious, sentient being) solely bears + the whole responsibility for the contribution, + they must understand the problem, and the solution, + and be able to constructively argue about it. + "well, AI said so, therefore it is" is not an acceptable approach. + +[^1]: Unless explicitly allowed by maintainers on case-by-case basis + *before* the contribution is submitted, + in which case the bot owner (conscious, sentient being) + is recognized as the de facto contributor. + E.g. https://github.com/apps/dependabot is allowed. diff --git a/AUTHORS b/AUTHORS index 5a39872fe0..ab26bb3ad5 100644 --- a/AUTHORS +++ b/AUTHORS @@ -10,9 +10,12 @@ Albert Pretorius Alex Steele +Alyssa Ross Andriy Berestovskyy Arne Beer +Benjamin King Carto +Cezary Skrzyński Christian Wassermann Christopher Seymour Colin Braley @@ -27,10 +30,15 @@ Eric Backus Eric Fiselier Eugene Zhuk Evgeny Safronov +Fabien Pichot Federico Ficarelli Felix Homann +Felmon Fekadu +Gergely Meszaros Gergő Szitár Google Inc. +Haihan Jiang +Henrique Bucher International Business Machines Corporation Ismael Jimenez Martinez Jern-Kuan Leong @@ -40,25 +48,35 @@ Jordan Williams Jussi Knuuttila Kaito Udagawa Kishan Kumar +Kostiantyn Lazukin Lei Xu +Marcel Jacobse Matt Clarkson Maxim Vafin +Mike Apodaca +Min-Yih Hsu MongoDB Inc. Nick Hutchinson Norman Heino Oleksandr Sochka +Olga Fadeeva Ori Livneh Paul Redmond +Prithvi Rao +qorexdev Radoslav Yovchev +Raghu Raja +Rainer Orth Roman Lebedev Sayan Bhattacharjee Shapr3D +Shashank Thakur Shuo Chen Staffan Tjernstrom Steinar H. Gunderson Stripe, Inc. Tobias Schmidt +XiaoWei Lu Yixuan Qiu Yusuke Suzuki Zbigniew Skowron -Min-Yih Hsu diff --git a/BUILD.bazel b/BUILD.bazel index 872adb0ff6..8cad9d6ce3 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,28 +1,39 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + licenses(["notice"]) -load("//:config/generate_export_header.bzl", "generate_export_header") +COPTS = [ + "-pedantic", + "-pedantic-errors", + "-std=c++17", + "-Wall", + "-Wconversion", + "-Wextra", + "-Wshadow", + # "-Wshorten-64-to-32", + "-Wfloat-equal", + "-Wformat=2", + "-fstrict-aliasing", + ## assert() are used a lot in tests upstream, which may be optimised out leading to + ## unused-variable warning. + "-Wno-unused-variable", + "-Werror=old-style-cast", +] -# Generate header to provide ABI export symbols -generate_export_header( - out = "include/benchmark/export.h", - lib = "benchmark", - static_define = "BENCHMARK_STATIC_DEFINE", -) +MSVC_COPTS = [ + "/std:c++17", +] config_setting( - name = "qnx", - constraint_values = ["@platforms//os:qnx"], - values = { - "cpu": "x64_qnx", - }, + name = "windows", + constraint_values = ["@platforms//os:windows"], visibility = [":__subpackages__"], ) config_setting( - name = "windows", - constraint_values = ["@platforms//os:windows"], - values = { - "cpu": "x64_windows", + name = "perfcounters", + define_values = { + "pfm": "1", }, visibility = [":__subpackages__"], ) @@ -38,26 +49,71 @@ cc_library( ), hdrs = [ "include/benchmark/benchmark.h", - "include/benchmark/export.h", # From generate_export_header + "include/benchmark/benchmark_api.h", + "include/benchmark/counter.h", + "include/benchmark/export.h", + "include/benchmark/macros.h", + "include/benchmark/managers.h", + "include/benchmark/registration.h", + "include/benchmark/reporter.h", + "include/benchmark/state.h", + "include/benchmark/statistics.h", + "include/benchmark/sysinfo.h", + "include/benchmark/types.h", + "include/benchmark/utils.h", ], + copts = select({ + ":windows": MSVC_COPTS, + "//conditions:default": COPTS, + }), + defines = [ + "BENCHMARK_STATIC_DEFINE", + "BENCHMARK_VERSION=\\\"" + (module_version() if module_version() != None else "") + "\\\"", + ] + select({ + ":perfcounters": ["HAVE_LIBPFM"], + "//conditions:default": [], + }), + includes = ["include"], linkopts = select({ ":windows": ["-DEFAULTLIB:shlwapi.lib"], "//conditions:default": ["-pthread"], }), - strip_include_prefix = "include", - visibility = ["//visibility:public"], # Only static linking is allowed; no .so will be produced. # Using `defines` (i.e. not `local_defines`) means that no # dependent rules need to bother about defining the macro. linkstatic = True, - defines = ["BENCHMARK_STATIC_DEFINE"], + local_defines = [ + # Turn on Large-file Support + "_FILE_OFFSET_BITS=64", + "_LARGEFILE64_SOURCE", + "_LARGEFILE_SOURCE", + ], + visibility = ["//visibility:public"], + deps = select({ + ":perfcounters": ["@libpfm"], + "//conditions:default": [], + }), ) cc_library( name = "benchmark_main", srcs = ["src/benchmark_main.cc"], - hdrs = ["include/benchmark/benchmark.h"], - strip_include_prefix = "include", + hdrs = [ + "include/benchmark/benchmark.h", + "include/benchmark/benchmark_api.h", + "include/benchmark/counter.h", + "include/benchmark/export.h", + "include/benchmark/macros.h", + "include/benchmark/managers.h", + "include/benchmark/registration.h", + "include/benchmark/reporter.h", + "include/benchmark/state.h", + "include/benchmark/statistics.h", + "include/benchmark/sysinfo.h", + "include/benchmark/types.h", + "include/benchmark/utils.h", + ], + includes = ["include"], visibility = ["//visibility:public"], deps = [":benchmark"], ) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6880c1f362..3031e6a7f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,19 +1,7 @@ -cmake_minimum_required (VERSION 3.5.1) - -foreach(p - CMP0048 # OK to clear PROJECT_VERSION on project() - CMP0054 # CMake 3.1 - CMP0056 # export EXE_LINKER_FLAGS to try_run - CMP0057 # Support no if() IN_LIST operator - CMP0063 # Honor visibility properties for all targets - CMP0077 # Allow option() overrides in importing projects - ) - if(POLICY ${p}) - cmake_policy(SET ${p} NEW) - endif() -endforeach() +# Require CMake 3.13. If available, use the policies up to CMake 3.22. +cmake_minimum_required (VERSION 3.13...3.22) -project (benchmark VERSION 1.6.1 LANGUAGES CXX) +project (benchmark VERSION 1.9.5 LANGUAGES CXX) option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON) option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the benchmark library." ON) @@ -26,11 +14,14 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "PGI") # PGC++ maybe reporting false positives. set(BENCHMARK_ENABLE_WERROR OFF) endif() +if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "NVHPC") + set(BENCHMARK_ENABLE_WERROR OFF) +endif() if(BENCHMARK_FORCE_WERROR) set(BENCHMARK_ENABLE_WERROR ON) endif(BENCHMARK_FORCE_WERROR) -if(NOT MSVC) +if(NOT (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")) option(BENCHMARK_BUILD_32_BITS "Build a 32 bit version of the library." OFF) else() set(BENCHMARK_BUILD_32_BITS OFF CACHE BOOL "Build a 32 bit version of the library - unsupported when using MSVC)" FORCE) @@ -38,6 +29,7 @@ endif() option(BENCHMARK_ENABLE_INSTALL "Enable installation of benchmark. (Projects embedding benchmark may want to turn this OFF.)" ON) option(BENCHMARK_ENABLE_DOXYGEN "Build documentation with Doxygen." OFF) option(BENCHMARK_INSTALL_DOCS "Enable installation of documentation." ON) +option(BENCHMARK_INSTALL_TOOLS "Enable installation of tools." ON) # Allow unmet dependencies to be met using CMake's ExternalProject mechanics, which # may require downloading the source code. @@ -49,12 +41,13 @@ option(BENCHMARK_ENABLE_GTEST_TESTS "Enable building the unit tests which depend option(BENCHMARK_USE_BUNDLED_GTEST "Use bundled GoogleTest. If disabled, the find_package(GTest) will be used." ON) option(BENCHMARK_ENABLE_LIBPFM "Enable performance counters provided by libpfm" OFF) +option(BENCHMARK_ENABLE_RUST_BINDINGS "Enable testing of the Rust bindings" OFF) # Export only public symbols -set(CMAKE_CXX_VISIBILITY_PRESET hidden) -set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) +#set(CMAKE_CXX_VISIBILITY_PRESET hidden) +#set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) -if(MSVC) +if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") # As of CMake 3.18, CMAKE_SYSTEM_PROCESSOR is not set properly for MSVC and # cross-compilation (e.g. Host=x86_64, target=aarch64) requires using the # undocumented, but working variable. @@ -75,7 +68,7 @@ function(should_enable_assembly_tests) return() endif() endif() - if (MSVC) + if (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC") return() elseif(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") return() @@ -113,37 +106,66 @@ get_git_version(GIT_VERSION) # If no git version can be determined, use the version # from the project() command -if ("${GIT_VERSION}" STREQUAL "0.0.0") - set(VERSION "${benchmark_VERSION}") +if ("${GIT_VERSION}" STREQUAL "v0.0.0") + set(VERSION "v${benchmark_VERSION}") else() set(VERSION "${GIT_VERSION}") endif() + +# Normalize version: drop "v" prefix, replace first "-" with ".", +# drop everything after second "-" (including said "-"). +string(STRIP ${VERSION} VERSION) +if(VERSION MATCHES v[^-]*-) + string(REGEX REPLACE "v([^-]*)-([0-9]+)-.*" "\\1.\\2" NORMALIZED_VERSION ${VERSION}) +else() + string(REGEX REPLACE "v(.*)" "\\1" NORMALIZED_VERSION ${VERSION}) +endif() + # Tell the user what versions we are using -message(STATUS "Version: ${VERSION}") +message(STATUS "Google Benchmark version: ${VERSION}, normalized to ${NORMALIZED_VERSION}") # The version of the libraries -set(GENERIC_LIB_VERSION ${VERSION}) -string(SUBSTRING ${VERSION} 0 1 GENERIC_LIB_SOVERSION) +set(GENERIC_LIB_VERSION ${NORMALIZED_VERSION}) +string(SUBSTRING ${NORMALIZED_VERSION} 0 1 GENERIC_LIB_SOVERSION) # Import our CMake modules include(AddCXXCompilerFlag) include(CheckCXXCompilerFlag) include(CheckLibraryExists) include(CXXFeatureCheck) -include(GenerateExportHeader) -check_library_exists(rt shm_open "" HAVE_LIB_RT) +# Check for rt library, but explicitly disable for QNX +if(QNXNTO) + set(HAVE_LIB_RT FALSE) +else() + check_library_exists(rt shm_open "" HAVE_LIB_RT) +endif() if (BENCHMARK_BUILD_32_BITS) add_required_cxx_compiler_flag(-m32) endif() +set(BENCHMARK_CXX_STANDARD 17) + +set(CMAKE_CXX_STANDARD ${BENCHMARK_CXX_STANDARD}) +set(CMAKE_CXX_STANDARD_REQUIRED YES) +set(CMAKE_CXX_EXTENSIONS OFF) + if (MSVC) # Turn compiler warnings up to 11 string(REGEX REPLACE "[-/]W[1-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4") + + # MP flag only applies to cl, not cl frontends to other compilers (e.g. clang-cl, icx-cl etc) + if(CMAKE_CXX_COMPILER_ID MATCHES MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP") + endif() add_definitions(-D_CRT_SECURE_NO_WARNINGS) + if(BENCHMARK_ENABLE_WERROR) + add_cxx_compiler_flag(-WX) + endif() + if (NOT BENCHMARK_ENABLE_EXCEPTIONS) add_cxx_compiler_flag(-EHs-) add_cxx_compiler_flag(-EHa-) @@ -170,66 +192,64 @@ if (MSVC) set(CMAKE_EXE_LINKER_FLAGS_MINSIZEREL "${CMAKE_EXE_LINKER_FLAGS_MINSIZEREL} /LTCG") endif() else() - # Try and enable C++11. Don't use C++14 because it doesn't work in some - # configurations. - add_cxx_compiler_flag(-std=c++11) - if (NOT HAVE_CXX_FLAG_STD_CXX11) - add_cxx_compiler_flag(-std=c++0x) - endif() - + # Turn on Large-file Support + add_definitions(-D_FILE_OFFSET_BITS=64) + add_definitions(-D_LARGEFILE64_SOURCE) + add_definitions(-D_LARGEFILE_SOURCE) # Turn compiler warnings up to 11 add_cxx_compiler_flag(-Wall) add_cxx_compiler_flag(-Wextra) add_cxx_compiler_flag(-Wshadow) add_cxx_compiler_flag(-Wfloat-equal) + add_cxx_compiler_flag(-Wold-style-cast) + add_cxx_compiler_flag(-Wconversion) + add_cxx_compiler_flag(-Wformat=2) if(BENCHMARK_ENABLE_WERROR) - add_cxx_compiler_flag(-Werror RELEASE) - add_cxx_compiler_flag(-Werror RELWITHDEBINFO) - add_cxx_compiler_flag(-Werror MINSIZEREL) + add_cxx_compiler_flag(-Werror) + add_cxx_compiler_flag(-pedantic-errors) endif() if (NOT BENCHMARK_ENABLE_TESTING) # Disable warning when compiling tests as gtest does not use 'override'. add_cxx_compiler_flag(-Wsuggest-override) endif() add_cxx_compiler_flag(-pedantic) - add_cxx_compiler_flag(-pedantic-errors) add_cxx_compiler_flag(-Wshorten-64-to-32) add_cxx_compiler_flag(-fstrict-aliasing) # Disable warnings regarding deprecated parts of the library while building # and testing those parts of the library. add_cxx_compiler_flag(-Wno-deprecated-declarations) - if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") # Intel silently ignores '-Wno-deprecated-declarations', # warning no. 1786 must be explicitly disabled. # See #631 for rationale. add_cxx_compiler_flag(-wd1786) + add_cxx_compiler_flag(-fno-finite-math-only) + # ICC17u2: overloaded virtual function "benchmark::Fixture::SetUp" is only partially + # overridden (because of deprecated overload) + add_cxx_compiler_flag(-wd654) endif() # Disable deprecation warnings for release builds (when -Werror is enabled). if(BENCHMARK_ENABLE_WERROR) - add_cxx_compiler_flag(-Wno-deprecated RELEASE) - add_cxx_compiler_flag(-Wno-deprecated RELWITHDEBINFO) - add_cxx_compiler_flag(-Wno-deprecated MINSIZEREL) + add_cxx_compiler_flag(-Wno-deprecated) endif() if (NOT BENCHMARK_ENABLE_EXCEPTIONS) add_cxx_compiler_flag(-fno-exceptions) endif() if (HAVE_CXX_FLAG_FSTRICT_ALIASING) - if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") #ICC17u2: Many false positives for Wstrict-aliasing + if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel" AND NOT CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") #ICC17u2: Many false positives for Wstrict-aliasing add_cxx_compiler_flag(-Wstrict-aliasing) endif() endif() - # ICC17u2: overloaded virtual function "benchmark::Fixture::SetUp" is only partially overridden - # (because of deprecated overload) - add_cxx_compiler_flag(-wd654) + add_cxx_compiler_flag(-Wthread-safety) if (HAVE_CXX_FLAG_WTHREAD_SAFETY) - cxx_feature_check(THREAD_SAFETY_ATTRIBUTES) + cxx_feature_check(THREAD_SAFETY_ATTRIBUTES "-DINCLUDE_DIRECTORIES=${PROJECT_SOURCE_DIR}/include") endif() # On most UNIX like platforms g++ and clang++ define _GNU_SOURCE as a # predefined macro, which turns on all of the wonderful libc extensions. - # However g++ doesn't do this in Cygwin so we have to define it ourselfs + # However g++ doesn't do this in Cygwin so we have to define it ourselves # since we depend on GNU/POSIX/BSD extensions. if (CYGWIN) add_definitions(-D_GNU_SOURCE=1) @@ -280,7 +300,8 @@ if (BENCHMARK_USE_LIBCXX) if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") add_cxx_compiler_flag(-stdlib=libc++) elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" OR - "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") + "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel" OR + "${CMAKE_CXX_COMPILER_ID}" STREQUAL "IntelLLVM") add_cxx_compiler_flag(-nostdinc++) message(WARNING "libc++ header path must be manually specified using CMAKE_CXX_FLAGS") # Adding -nodefaultlibs directly to CMAKE__LINKER_FLAGS will break @@ -294,17 +315,11 @@ if (BENCHMARK_USE_LIBCXX) endif() endif(BENCHMARK_USE_LIBCXX) -set(EXTRA_CXX_FLAGS "") -if (WIN32 AND "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") - # Clang on Windows fails to compile the regex feature check under C++11 - set(EXTRA_CXX_FLAGS "-DCMAKE_CXX_STANDARD=14") -endif() - # C++ feature checks # Determine the correct regular expression engine to use -cxx_feature_check(STD_REGEX ${EXTRA_CXX_FLAGS}) -cxx_feature_check(GNU_POSIX_REGEX ${EXTRA_CXX_FLAGS}) -cxx_feature_check(POSIX_REGEX ${EXTRA_CXX_FLAGS}) +cxx_feature_check(STD_REGEX) +cxx_feature_check(GNU_POSIX_REGEX) +cxx_feature_check(POSIX_REGEX) if(NOT HAVE_STD_REGEX AND NOT HAVE_GNU_POSIX_REGEX AND NOT HAVE_POSIX_REGEX) message(FATAL_ERROR "Failed to determine the source files for the regular expression backend") endif() @@ -318,8 +333,15 @@ cxx_feature_check(STEADY_CLOCK) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) +# cxx_feature_check relies on try_run to probe compiler features. Because this +# check does not produce a real target, target_link_libraries cannot be applied. +# Therefore, link libraries are forwarded to try_run through +# BENCHMARK_CXX_LIBRARIES. +list(APPEND BENCHMARK_CXX_LIBRARIES Threads::Threads) +cxx_feature_check(PTHREAD_AFFINITY) + if (BENCHMARK_ENABLE_LIBPFM) - find_package(PFM) + find_package(PFM REQUIRED) endif() # Set up directories diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 43de4c9d47..378c064125 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,6 +53,11 @@ of your first [pull request][]. 1. Finally, push the commits to your fork and submit a [pull request][]. + +## AI usage ## + +Please consult [AGENTS](AGENTS.md) file. + [forking]: https://help.github.com/articles/fork-a-repo [well-formed commit messages]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html [pull request]: https://help.github.com/articles/creating-a-pull-request diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 35a4cc66f0..88c3b1b327 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -25,10 +25,13 @@ Abhina Sreeskantharajan Albert Pretorius Alex Steele +Alyssa Ross Andriy Berestovskyy Arne Beer Bátor Tallér +Benjamin King Billy Robert O'Neal III +Cezary Skrzyński Chris Kennelly Christian Wassermann Christopher Seymour @@ -41,48 +44,66 @@ Dominic Hamon Dominik Czarnota Dominik Korman Donald Aingworth +Doug Evans Eric Backus Eric Fiselier Eugene Zhuk Evgeny Safronov +Fabien Pichot Fanbo Meng Federico Ficarelli Felix Homann +Felmon Fekadu Geoffrey Martin-Noble +Gergely Meszaros Gergő Szitár +Haihan Jiang Hannes Hauswedell +Henrique Bucher Ismael Jimenez Martinez +Iakov Sergeev Jern-Kuan Leong JianXiong Zhou Joao Paulo Magalhaes John Millikin Jordan Williams Jussi Knuuttila -Kai Wolf Kaito Udagawa +Kai Wolf Kishan Kumar +Kostiantyn Lazukin Lei Xu +Marcel Jacobse Matt Clarkson Maxim Vafin +Mike Apodaca +Min-Yih Hsu Nick Hutchinson Norman Heino Oleksandr Sochka +Olga Fadeeva Ori Livneh Pascal Leroy Paul Redmond Pierre Phaneuf +Prithvi Rao +qorexdev Radoslav Yovchev +Raghu Raja +Rainer Orth Raul Marin Ray Glover Robert Guo Roman Lebedev Sayan Bhattacharjee +Shashank Thakur +ShengYi Hung Shuo Chen Steven Wan Tobias Schmidt Tobias Ulvgård Tom Madams +XiaoWei Lu Yixuan Qiu Yusuke Suzuki Zbigniew Skowron -Min-Yih Hsu diff --git a/LICENSE b/LICENSE index a5c40b3ee1..d645695673 100644 --- a/LICENSE +++ b/LICENSE @@ -200,35 +200,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - -Only benchmark/config/generate_export_header.bzl depends on the following licence: - - BSD 3-Clause License - -Copyright (c) [year], [fullname] - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/MODULE.bazel b/MODULE.bazel new file mode 100644 index 0000000000..ad3df4a4f5 --- /dev/null +++ b/MODULE.bazel @@ -0,0 +1,41 @@ +module( + name = "google_benchmark", + version = "1.9.5", +) + +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") +bazel_dep(name = "rules_cc", version = "0.0.9") + +bazel_dep(name = "rules_python", version = "1.0.0", dev_dependency = True) +bazel_dep(name = "googletest", version = "1.14.0", dev_dependency = True, repo_name = "com_google_googletest") + +bazel_dep(name = "libpfm", version = "4.11.0.bcr.1") + +# Register a toolchain for Python 3.9 to be able to build numpy. Python +# versions >=3.10 are problematic. +# A second reason for this is to be able to build Python hermetically instead +# of relying on the changing default version from rules_python. + +python = use_extension("@rules_python//python/extensions:python.bzl", "python", dev_dependency = True) +python.toolchain(python_version = "3.8") +python.toolchain(python_version = "3.9") +python.toolchain(python_version = "3.10") +python.toolchain(python_version = "3.11") +python.toolchain( + is_default = True, + python_version = "3.12", +) +python.toolchain(python_version = "3.13") + +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip", dev_dependency = True) +pip.parse( + hub_name = "tools_pip_deps", + python_version = "3.12", + requirements_lock = "//tools:requirements.txt", +) +use_repo(pip, "tools_pip_deps") + +# -- bazel_dep definitions -- # + +bazel_dep(name = "nanobind_bazel", version = "2.9.2", dev_dependency = True) diff --git a/README.md b/README.md index 205fb008af..77a2336d96 100644 --- a/README.md +++ b/README.md @@ -2,17 +2,17 @@ [![build-and-test](https://github.com/google/benchmark/workflows/build-and-test/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Abuild-and-test) [![bazel](https://github.com/google/benchmark/actions/workflows/bazel.yml/badge.svg)](https://github.com/google/benchmark/actions/workflows/bazel.yml) -[![pylint](https://github.com/google/benchmark/workflows/pylint/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Apylint) [![test-bindings](https://github.com/google/benchmark/workflows/test-bindings/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Atest-bindings) - -[![Build Status](https://travis-ci.org/google/benchmark.svg?branch=master)](https://travis-ci.org/google/benchmark) [![Coverage Status](https://coveralls.io/repos/google/benchmark/badge.svg)](https://coveralls.io/r/google/benchmark) +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/google/benchmark/badge)](https://securityscorecards.dev/viewer/?uri=github.com/google/benchmark) +[![Discord](https://discordapp.com/api/guilds/1125694995928719494/widget.png?style=shield)](https://discord.gg/cz7UX7wKC2) A library to benchmark code snippets, similar to unit tests. Example: ```c++ -#include +#include +#include static void BM_SomeFunction(benchmark::State& state) { // Perform setup here @@ -33,7 +33,7 @@ To get started, see [Requirements](#requirements) and [Installation](#installation). See [Usage](#usage) for a full example and the [User Guide](docs/user_guide.md) for a more comprehensive feature overview. -It may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/master/docs/primer.md) +It may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/main/docs/primer.md) as some of the structural aspects of the APIs are similar. ## Resources @@ -51,15 +51,13 @@ IRC channels: ## Requirements -The library can be used with C++03. However, it requires C++11 to build, +The library can be used with C++11. However, it requires C++17 to build, including compiler and standard library support. -The following minimum versions are required to build the library: +_See [dependencies.md](docs/dependencies.md) for more details regarding supported +compilers and standards._ -* GCC 4.8 -* Clang 3.4 -* Visual Studio 14 2015 -* Intel 2015 Update 1 +If you have need for a particular compiler to be supported, patches are very welcome. See [Platform-Specific Build Instructions](docs/platform_specific_build_instructions.md). @@ -79,9 +77,7 @@ $ cd benchmark # Make a build directory to place the build output. $ cmake -E make_directory "build" # Generate build system files with cmake, and download any dependencies. -$ cmake -E chdir "build" cmake -DBENCHMARK_DOWNLOAD_DEPENDENCIES=on -DCMAKE_BUILD_TYPE=Release ../ -# or, starting with CMake 3.13, use a simpler form: -# cmake -DCMAKE_BUILD_TYPE=Release -S . -B "build" +$ cmake -DBENCHMARK_DOWNLOAD_DEPENDENCIES=on -DCMAKE_BUILD_TYPE=Release -S . -B "build" # Build the library. $ cmake --build "build" --config Release ``` @@ -139,6 +135,12 @@ cache variables, if autodetection fails. If you are using clang, you may need to set `LLVMAR_EXECUTABLE`, `LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables. +To enable sanitizer checks (eg., `asan` and `tsan`), add: +``` + -DCMAKE_C_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=address -fsanitize=thread -fno-sanitize-recover=all" + -DCMAKE_CXX_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=address -fsanitize=thread -fno-sanitize-recover=all " +``` + ### Stable and Experimental Library Versions The main branch contains the latest stable version of the benchmarking library; @@ -200,8 +202,11 @@ flag for option information or see the [User Guide](docs/user_guide.md). ### Usage with CMake If using CMake, it is recommended to link against the project-provided -`benchmark::benchmark` and `benchmark::benchmark_main` targets using -`target_link_libraries`. +`benchmark::benchmark` or `benchmark::benchmark_main` targets using +`target_link_libraries`. Link to `benchmark::benchmark` when your target +defines its own `main` function, or link to `benchmark::benchmark_main` to use +the default benchmark entry point. The `benchmark::benchmark_main` target links +`benchmark::benchmark` transitively. It is possible to use ```find_package``` to import an installed version of the library. ```cmake @@ -215,4 +220,85 @@ add_subdirectory(benchmark) Either way, link to the library as follows. ```cmake target_link_libraries(MyTarget benchmark::benchmark) +# Or, when you do not define your own main: +target_link_libraries(MyTarget benchmark::benchmark_main) +``` +When benchmark sources are shared through an intermediate CMake target, choose +an object library instead of a static library: + +```cmake +add_library(shared_benchmarks OBJECT bench.cc) +target_link_libraries(shared_benchmarks benchmark::benchmark_main) +add_executable(runnable_benchmarks) +target_link_libraries(runnable_benchmarks shared_benchmarks) +``` + +This links the object file that contains `BENCHMARK` registrations into the +final executable. If those registrations are placed only in an intermediate +`STATIC` library, the linker may not copy static registration symbols, and thus +benchmarks will not be part of the final executable. + +#### Embedding Google Benchmark in another CMake project + +There are two common ways to consume Google Benchmark from a CMake project: + +* Use an installed or package-managed copy, for example from a system package + manager or vcpkg, and import it with `find_package(benchmark REQUIRED)`. +* Add this repository to the source tree, for example as a submodule or + `FetchContent` dependency, and call `add_subdirectory`. + +The installed form keeps Google Benchmark's build separate from the parent +project and is usually the simplest choice for system packages and vcpkg. The +source-tree form is useful when the parent project wants to pin a specific +commit or build Google Benchmark as part of its normal CMake configure step. + +When embedding from source, most projects should turn off Google Benchmark's +tests and install rules: + +```cmake +set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "" FORCE) +set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) +set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "" FORCE) +add_subdirectory(third_party/benchmark) +target_link_libraries(MyTarget benchmark::benchmark) +``` + +If Google Test is not already provided by the parent build, either check out the +Google Test sources under `benchmark/googletest` or configure with +`BENCHMARK_DOWNLOAD_DEPENDENCIES=ON`. For projects that only link the benchmark +library and do not build Google Benchmark's tests, disabling +`BENCHMARK_ENABLE_GTEST_TESTS` avoids the Google Test dependency. + +Google Benchmark follows CMake's `BUILD_SHARED_LIBS` setting when selecting +static or shared library output. On Windows, keep this setting consistent with +the rest of the project and make sure the same runtime library configuration is +used across the benchmark library and the targets that link it. + +### Usage with Bazel + +If using Bazel with Bzlmod, add Google Benchmark to your `MODULE.bazel` file: + +```starlark +bazel_dep(name = "google_benchmark", version = "") +``` + +Replace `` with the Google Benchmark release version you want to use. + +Then link a `cc_binary` or `cc_test` against one of the provided targets: + +```starlark +load("@rules_cc//cc:defs.bzl", "cc_binary") + +cc_binary( + name = "my_benchmark", + srcs = ["my_benchmark.cc"], + deps = ["@google_benchmark//:benchmark_main"], +) ``` + +Use `@google_benchmark//:benchmark` when your target defines its own `main` +function, including through `BENCHMARK_MAIN()`. Use +`@google_benchmark//:benchmark_main` to use the default Google Benchmark entry +point. + +For WORKSPACE setup and more examples, see [Bazel](docs/bazel.md). diff --git a/WORKSPACE b/WORKSPACE index 949eb98bc5..dca4850930 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,44 +1,20 @@ workspace(name = "com_github_google_benchmark") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -http_archive( - name = "com_google_absl", - sha256 = "f41868f7a938605c92936230081175d1eae87f6ea2c248f41077c8f88316f111", - strip_prefix = "abseil-cpp-20200225.2", - urls = ["https://github.com/abseil/abseil-cpp/archive/20200225.2.tar.gz"], -) +load("//:bazel/benchmark_deps.bzl", "benchmark_deps") -git_repository( - name = "com_google_googletest", - remote = "https://github.com/google/googletest.git", - tag = "release-1.11.0", -) +benchmark_deps() -http_archive( - name = "pybind11", - build_file = "@//bindings/python:pybind11.BUILD", - sha256 = "1eed57bc6863190e35637290f97a20c81cfe4d9090ac0a24f3bbf08f265eb71d", - strip_prefix = "pybind11-2.4.3", - urls = ["https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz"], -) +load("@rules_python//python:repositories.bzl", "py_repositories") -new_local_repository( - name = "python_headers", - build_file = "@//bindings/python:python_headers.BUILD", - path = "/usr/include/python3.6", # May be overwritten by setup.py. -) +py_repositories() + +load("@rules_python//python:pip.bzl", "pip_parse") -http_archive( - name = "rules_python", - url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz", - sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0", +pip_parse( + name = "tools_pip_deps", + requirements_lock = "//tools:requirements.txt", ) -load("@rules_python//python:pip.bzl", pip3_install="pip_install") +load("@tools_pip_deps//:requirements.bzl", "install_deps") -pip3_install( - name = "py_deps", - requirements = "//:requirements.txt", -) +install_deps() diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod new file mode 100644 index 0000000000..9526376d77 --- /dev/null +++ b/WORKSPACE.bzlmod @@ -0,0 +1,2 @@ +# This file marks the root of the Bazel workspace. +# See MODULE.bazel for dependencies and setup. diff --git a/bazel/benchmark_deps.bzl b/bazel/benchmark_deps.bzl new file mode 100644 index 0000000000..a6be602413 --- /dev/null +++ b/bazel/benchmark_deps.bzl @@ -0,0 +1,54 @@ +""" +This file contains the Bazel build dependencies for Google Benchmark (both C++ source and Python bindings). +""" + +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def benchmark_deps(): + """Loads dependencies required to build Google Benchmark.""" + + if "bazel_skylib" not in native.existing_rules(): + http_archive( + name = "bazel_skylib", + sha256 = "cd55a062e763b9349921f0f5db8c3933288dc8ba4f76dd9416aac68acee3cb94", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz", + ], + ) + + if "rules_python" not in native.existing_rules(): + http_archive( + name = "rules_python", + sha256 = "e85ae30de33625a63eca7fc40a94fea845e641888e52f32b6beea91e8b1b2793", + strip_prefix = "rules_python-0.27.1", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.27.1/rules_python-0.27.1.tar.gz", + ) + + if "com_google_googletest" not in native.existing_rules(): + new_git_repository( + name = "com_google_googletest", + remote = "https://github.com/google/googletest.git", + tag = "release-1.12.1", + ) + + if "nanobind" not in native.existing_rules(): + new_git_repository( + name = "nanobind", + remote = "https://github.com/wjakob/nanobind.git", + tag = "v1.9.2", + build_file = "@//bindings/python:nanobind.BUILD", + recursive_init_submodules = True, + ) + + if "libpfm" not in native.existing_rules(): + # Downloaded from v4.9.0 tag at https://sourceforge.net/p/perfmon2/libpfm4/ref/master/tags/ + http_archive( + name = "libpfm", + build_file = str(Label("//tools:libpfm.BUILD.bazel")), + sha256 = "5da5f8872bde14b3634c9688d980f68bda28b510268723cc12973eedbab9fecc", + type = "tar.gz", + strip_prefix = "libpfm-4.11.0", + urls = ["https://sourceforge.net/projects/perfmon2/files/libpfm4/libpfm-4.11.0.tar.gz/download"], + ) diff --git a/bindings/python/BUILD b/bindings/python/BUILD deleted file mode 100644 index 9559a76b30..0000000000 --- a/bindings/python/BUILD +++ /dev/null @@ -1,3 +0,0 @@ -exports_files(glob(["*.BUILD"])) -exports_files(["build_defs.bzl"]) - diff --git a/bindings/python/build_defs.bzl b/bindings/python/build_defs.bzl deleted file mode 100644 index 009820afd0..0000000000 --- a/bindings/python/build_defs.bzl +++ /dev/null @@ -1,25 +0,0 @@ -_SHARED_LIB_SUFFIX = { - "//conditions:default": ".so", - "//:windows": ".dll", -} - -def py_extension(name, srcs, hdrs = [], copts = [], features = [], deps = []): - for shared_lib_suffix in _SHARED_LIB_SUFFIX.values(): - shared_lib_name = name + shared_lib_suffix - native.cc_binary( - name = shared_lib_name, - linkshared = True, - linkstatic = True, - srcs = srcs + hdrs, - copts = copts, - features = features, - deps = deps, - ) - - return native.py_library( - name = name, - data = select({ - platform: [name + shared_lib_suffix] - for platform, shared_lib_suffix in _SHARED_LIB_SUFFIX.items() - }), - ) diff --git a/bindings/python/google_benchmark/BUILD b/bindings/python/google_benchmark/BUILD index 3c1561f48e..8938b3716e 100644 --- a/bindings/python/google_benchmark/BUILD +++ b/bindings/python/google_benchmark/BUILD @@ -1,4 +1,5 @@ -load("//bindings/python:build_defs.bzl", "py_extension") +load("@nanobind_bazel//:build_defs.bzl", "nanobind_extension", "nanobind_stubgen") +load("@rules_python//python:defs.bzl", "py_library", "py_test") py_library( name = "google_benchmark", @@ -6,23 +7,19 @@ py_library( visibility = ["//visibility:public"], deps = [ ":_benchmark", - # pip; absl:app ], ) -py_extension( +nanobind_extension( name = "_benchmark", srcs = ["benchmark.cc"], - copts = [ - "-fexceptions", - "-fno-strict-aliasing", - ], - features = ["-use_header_modules"], - deps = [ - "//:benchmark", - "@pybind11", - "@python_headers", - ], + deps = ["//:benchmark"], +) + +nanobind_stubgen( + name = "benchmark_stubgen", + marker_file = "bindings/python/google_benchmark/py.typed", + module = ":_benchmark", ) py_test( @@ -35,4 +32,3 @@ py_test( ":google_benchmark", ], ) - diff --git a/bindings/python/google_benchmark/__init__.py b/bindings/python/google_benchmark/__init__.py index ec651c14fb..331a88e9b5 100644 --- a/bindings/python/google_benchmark/__init__.py +++ b/bindings/python/google_benchmark/__init__.py @@ -27,46 +27,28 @@ def my_benchmark(state): benchmark.main() """ -from absl import app +import atexit + from google_benchmark import _benchmark from google_benchmark._benchmark import ( - Counter, - kNanosecond, - kMicrosecond, - kMillisecond, - kSecond, - oNone, - o1, - oN, - oNSquared, - oNCubed, - oLogN, - oNLogN, - oAuto, - oLambda, + Counter as Counter, + State as State, + kMicrosecond as kMicrosecond, + kMillisecond as kMillisecond, + kNanosecond as kNanosecond, + kSecond as kSecond, + o1 as o1, + oAuto as oAuto, + oLambda as oLambda, + oLogN as oLogN, + oN as oN, + oNCubed as oNCubed, + oNLogN as oNLogN, + oNone as oNone, + oNSquared as oNSquared, ) - -__all__ = [ - "register", - "main", - "Counter", - "kNanosecond", - "kMicrosecond", - "kMillisecond", - "kSecond", - "oNone", - "o1", - "oN", - "oNSquared", - "oNCubed", - "oLogN", - "oNLogN", - "oAuto", - "oLambda", -] - -__version__ = "1.6.1" +__version__ = "1.9.5" class __OptionMaker: @@ -76,7 +58,8 @@ class __OptionMaker: """ class Options: - """Pure data class to store options calls, along with the benchmarked function.""" + """Pure data class to store options calls, along with the benchmarked + function.""" def __init__(self, func): self.func = func @@ -94,14 +77,13 @@ def __getattr__(self, builder_name): # The function that get returned on @option.range(start=0, limit=1<<5). def __builder_method(*args, **kwargs): - # The decorator that get called, either with the benchmared function # or the previous Options def __decorator(func_or_options): options = self.make(func_or_options) options.builder_calls.append((builder_name, args, kwargs)) - # The decorator returns Options so it is not technically a decorator - # and needs a final call to @regiser + # The decorator returns Options so it is not technically a + # decorator and needs a final call to @register return options return __decorator @@ -110,8 +92,8 @@ def __decorator(func_or_options): # Alias for nicer API. -# We have to instantiate an object, even if stateless, to be able to use __getattr__ -# on option.range +# We have to instantiate an object, even if stateless, to be able to use +# __getattr__ on option.range option = __OptionMaker() @@ -121,8 +103,8 @@ def register(undefined=None, *, name=None): # Decorator is called without parenthesis so we return a decorator return lambda f: register(f, name=name) - # We have either the function to benchmark (simple case) or an instance of Options - # (@option._ case). + # We have either the function to benchmark (simple case) or an instance of + # Options (@option._ case). options = __OptionMaker.make(undefined) if name is None: @@ -138,21 +120,17 @@ def register(undefined=None, *, name=None): return options.func -def _flags_parser(argv): - argv = _benchmark.Initialize(argv) - return app.parse_flags_with_usage(argv) +def main(argv: list[str] | None = None) -> None: + import sys - -def _run_benchmarks(argv): - if len(argv) > 1: - raise app.UsageError("Too many command-line arguments.") + _benchmark.Initialize(argv or sys.argv) return _benchmark.RunSpecifiedBenchmarks() -def main(argv=None): - return app.run(_run_benchmarks, argv=argv, flags_parser=_flags_parser) - +# FIXME: can we rerun with disabled ASLR? # Methods for use with custom main function. initialize = _benchmark.Initialize run_benchmarks = _benchmark.RunSpecifiedBenchmarks +add_custom_context = _benchmark.AddCustomContext +atexit.register(_benchmark.ClearRegisteredBenchmarks) diff --git a/bindings/python/google_benchmark/benchmark.cc b/bindings/python/google_benchmark/benchmark.cc index ea40990e08..ccd7eb5a50 100644 --- a/bindings/python/google_benchmark/benchmark.cc +++ b/bindings/python/google_benchmark/benchmark.cc @@ -2,31 +2,30 @@ #include "benchmark/benchmark.h" -#include -#include -#include +#include "nanobind/nanobind.h" +#include "nanobind/operators.h" +#include "nanobind/stl/bind_map.h" +#include "nanobind/stl/string.h" +#include "nanobind/stl/vector.h" -#include "pybind11/operators.h" -#include "pybind11/pybind11.h" -#include "pybind11/stl.h" -#include "pybind11/stl_bind.h" - -PYBIND11_MAKE_OPAQUE(benchmark::UserCounters); +NB_MAKE_OPAQUE(benchmark::UserCounters); namespace { -namespace py = ::pybind11; +namespace nb = nanobind; std::vector Initialize(const std::vector& argv) { - // The `argv` pointers here become invalid when this function returns, but - // benchmark holds the pointer to `argv[0]`. We create a static copy of it - // so it persists, and replace the pointer below. - static std::string executable_name(argv[0]); std::vector ptrs; ptrs.reserve(argv.size()); for (auto& arg : argv) { ptrs.push_back(const_cast(arg.c_str())); } - ptrs[0] = const_cast(executable_name.c_str()); + if (!ptrs.empty()) { + // The `argv` pointers here become invalid when this function returns, but + // benchmark holds the pointer to `argv[0]`. We create a static copy of it + // so it persists, and replace the pointer below. + static std::string executable_name(argv[0]); + ptrs[0] = const_cast(executable_name.c_str()); + } int argc = static_cast(argv.size()); benchmark::Initialize(&argc, ptrs.data()); std::vector remaining_argv; @@ -37,15 +36,15 @@ std::vector Initialize(const std::vector& argv) { return remaining_argv; } -benchmark::internal::Benchmark* RegisterBenchmark(const char* name, - py::function f) { +benchmark::Benchmark* RegisterBenchmark(const std::string& name, + nb::callable f) { return benchmark::RegisterBenchmark( name, [f](benchmark::State& state) { f(&state); }); } -PYBIND11_MODULE(_benchmark, m) { +NB_MODULE(_benchmark, m) { using benchmark::TimeUnit; - py::enum_(m, "TimeUnit") + nb::enum_(m, "TimeUnit") .value("kNanosecond", TimeUnit::kNanosecond) .value("kMicrosecond", TimeUnit::kMicrosecond) .value("kMillisecond", TimeUnit::kMillisecond) @@ -53,74 +52,67 @@ PYBIND11_MODULE(_benchmark, m) { .export_values(); using benchmark::BigO; - py::enum_(m, "BigO") + nb::enum_(m, "BigO") .value("oNone", BigO::oNone) .value("o1", BigO::o1) .value("oN", BigO::oN) .value("oNSquared", BigO::oNSquared) .value("oNCubed", BigO::oNCubed) .value("oLogN", BigO::oLogN) - .value("oNLogN", BigO::oLogN) + .value("oNLogN", BigO::oNLogN) .value("oAuto", BigO::oAuto) .value("oLambda", BigO::oLambda) .export_values(); - using benchmark::internal::Benchmark; - py::class_(m, "Benchmark") - // For methods returning a pointer tor the current object, reference - // return policy is used to ask pybind not to take ownership oof the + using benchmark::Benchmark; + nb::class_(m, "Benchmark") + // For methods returning a pointer to the current object, reference + // return policy is used to ask nanobind not to take ownership of the // returned object and avoid calling delete on it. // https://pybind11.readthedocs.io/en/stable/advanced/functions.html#return-value-policies // // For methods taking a const std::vector<...>&, a copy is created // because a it is bound to a Python list. // https://pybind11.readthedocs.io/en/stable/advanced/cast/stl.html - .def("unit", &Benchmark::Unit, py::return_value_policy::reference) - .def("arg", &Benchmark::Arg, py::return_value_policy::reference) - .def("args", &Benchmark::Args, py::return_value_policy::reference) - .def("range", &Benchmark::Range, py::return_value_policy::reference, - py::arg("start"), py::arg("limit")) - .def("dense_range", &Benchmark::DenseRange, - py::return_value_policy::reference, py::arg("start"), - py::arg("limit"), py::arg("step") = 1) - .def("ranges", &Benchmark::Ranges, py::return_value_policy::reference) - .def("args_product", &Benchmark::ArgsProduct, - py::return_value_policy::reference) - .def("arg_name", &Benchmark::ArgName, py::return_value_policy::reference) - .def("arg_names", &Benchmark::ArgNames, - py::return_value_policy::reference) - .def("range_pair", &Benchmark::RangePair, - py::return_value_policy::reference, py::arg("lo1"), py::arg("hi1"), - py::arg("lo2"), py::arg("hi2")) + .def("unit", &Benchmark::Unit, nb::rv_policy::reference) + .def("arg", &Benchmark::Arg, nb::rv_policy::reference) + .def("args", &Benchmark::Args, nb::rv_policy::reference) + .def("range", &Benchmark::Range, nb::rv_policy::reference, + nb::arg("start"), nb::arg("limit")) + .def("dense_range", &Benchmark::DenseRange, nb::rv_policy::reference, + nb::arg("start"), nb::arg("limit"), nb::arg("step") = 1) + .def("ranges", &Benchmark::Ranges, nb::rv_policy::reference) + .def("args_product", &Benchmark::ArgsProduct, nb::rv_policy::reference) + .def("arg_name", &Benchmark::ArgName, nb::rv_policy::reference) + .def("arg_names", &Benchmark::ArgNames, nb::rv_policy::reference) + .def("range_pair", &Benchmark::RangePair, nb::rv_policy::reference, + nb::arg("lo1"), nb::arg("hi1"), nb::arg("lo2"), nb::arg("hi2")) .def("range_multiplier", &Benchmark::RangeMultiplier, - py::return_value_policy::reference) - .def("min_time", &Benchmark::MinTime, py::return_value_policy::reference) + nb::rv_policy::reference) + .def("min_time", &Benchmark::MinTime, nb::rv_policy::reference) .def("min_warmup_time", &Benchmark::MinWarmUpTime, - py::return_value_policy::reference) - .def("iterations", &Benchmark::Iterations, - py::return_value_policy::reference) - .def("repetitions", &Benchmark::Repetitions, - py::return_value_policy::reference) + nb::rv_policy::reference) + .def("iterations", &Benchmark::Iterations, nb::rv_policy::reference) + .def("repetitions", &Benchmark::Repetitions, nb::rv_policy::reference) .def("report_aggregates_only", &Benchmark::ReportAggregatesOnly, - py::return_value_policy::reference, py::arg("value") = true) + nb::rv_policy::reference, nb::arg("value") = true) .def("display_aggregates_only", &Benchmark::DisplayAggregatesOnly, - py::return_value_policy::reference, py::arg("value") = true) + nb::rv_policy::reference, nb::arg("value") = true) .def("measure_process_cpu_time", &Benchmark::MeasureProcessCPUTime, - py::return_value_policy::reference) - .def("use_real_time", &Benchmark::UseRealTime, - py::return_value_policy::reference) + nb::rv_policy::reference) + .def("use_real_time", &Benchmark::UseRealTime, nb::rv_policy::reference) .def("use_manual_time", &Benchmark::UseManualTime, - py::return_value_policy::reference) + nb::rv_policy::reference) .def( "complexity", (Benchmark * (Benchmark::*)(benchmark::BigO)) & Benchmark::Complexity, - py::return_value_policy::reference, - py::arg("complexity") = benchmark::oAuto); + nb::rv_policy::reference, nb::arg("complexity") = benchmark::oAuto); using benchmark::Counter; - py::class_ py_counter(m, "Counter"); + nb::class_ py_counter(m, "Counter"); - py::enum_(py_counter, "Flags") + nb::enum_(py_counter, "Flags", nb::is_arithmetic(), + nb::is_flag()) .value("kDefaults", Counter::Flags::kDefaults) .value("kIsRate", Counter::Flags::kIsRate) .value("kAvgThreads", Counter::Flags::kAvgThreads) @@ -131,53 +123,67 @@ PYBIND11_MODULE(_benchmark, m) { .value("kAvgIterations", Counter::Flags::kAvgIterations) .value("kAvgIterationsRate", Counter::Flags::kAvgIterationsRate) .value("kInvert", Counter::Flags::kInvert) - .export_values() - .def(py::self | py::self); + .export_values(); - py::enum_(py_counter, "OneK") + nb::enum_(py_counter, "OneK") .value("kIs1000", Counter::OneK::kIs1000) .value("kIs1024", Counter::OneK::kIs1024) .export_values(); py_counter - .def(py::init(), - py::arg("value") = 0., py::arg("flags") = Counter::kDefaults, - py::arg("k") = Counter::kIs1000) - .def(py::init([](double value) { return Counter(value); })) - .def_readwrite("value", &Counter::value) - .def_readwrite("flags", &Counter::flags) - .def_readwrite("oneK", &Counter::oneK); - py::implicitly_convertible(); - py::implicitly_convertible(); - - py::bind_map(m, "UserCounters"); + .def(nb::init(), + nb::arg("value") = 0., nb::arg("flags") = Counter::kDefaults, + nb::arg("k") = Counter::kIs1000) + .def("__init__", + ([](Counter* c, double value) { new (c) Counter(value); })) + .def_rw("value", &Counter::value) + .def_rw("flags", &Counter::flags) + .def_rw("oneK", &Counter::oneK) + .def(nb::init_implicit()); + + nb::implicitly_convertible(); + + nb::bind_map(m, "UserCounters"); using benchmark::State; - py::class_(m, "State") + nb::class_(m, "State") .def("__bool__", &State::KeepRunning) - .def_property_readonly("keep_running", &State::KeepRunning) + .def_prop_ro("keep_running", &State::KeepRunning) .def("pause_timing", &State::PauseTiming) .def("resume_timing", &State::ResumeTiming) .def("skip_with_error", &State::SkipWithError) - .def_property_readonly("error_occurred", &State::error_occurred) + .def_prop_ro("error_occurred", &State::error_occurred) .def("set_iteration_time", &State::SetIterationTime) - .def_property("bytes_processed", &State::bytes_processed, - &State::SetBytesProcessed) - .def_property("complexity_n", &State::complexity_length_n, - &State::SetComplexityN) - .def_property("items_processed", &State::items_processed, - &State::SetItemsProcessed) - .def("set_label", (void (State::*)(const char*)) & State::SetLabel) - .def("range", &State::range, py::arg("pos") = 0) - .def_property_readonly("iterations", &State::iterations) - .def_readwrite("counters", &State::counters) - .def_property_readonly("thread_index", &State::thread_index) - .def_property_readonly("threads", &State::threads); + .def_prop_rw("bytes_processed", &State::bytes_processed, + &State::SetBytesProcessed) + .def_prop_rw("complexity_n", &State::complexity_length_n, + &State::SetComplexityN) + .def_prop_rw("items_processed", &State::items_processed, + &State::SetItemsProcessed) + .def("set_label", &State::SetLabel) + .def( + "range", + [](const State& state, std::size_t pos = 0) -> int64_t { + if (pos < state.range_size()) { + return state.range(pos); + } + throw nb::index_error("pos is out of range"); + }, + nb::arg("pos") = 0) + .def_prop_ro("iterations", &State::iterations) + .def_prop_ro("name", &State::name) + .def_rw("counters", &State::counters) + .def_prop_ro("thread_index", &State::thread_index) + .def_prop_ro("threads", &State::threads); m.def("Initialize", Initialize); - m.def("RegisterBenchmark", RegisterBenchmark, - py::return_value_policy::reference); + m.def("RegisterBenchmark", RegisterBenchmark, nb::rv_policy::reference); m.def("RunSpecifiedBenchmarks", []() { benchmark::RunSpecifiedBenchmarks(); }); + m.def("ClearRegisteredBenchmarks", benchmark::ClearRegisteredBenchmarks); + m.def("AddCustomContext", benchmark::AddCustomContext, nb::arg("key"), + nb::arg("value"), + "Add a key-value pair to output as part of the context stanza in the " + "report."); }; } // namespace diff --git a/bindings/python/google_benchmark/example.py b/bindings/python/google_benchmark/example.py index 487acc9f1e..8217b409e0 100644 --- a/bindings/python/google_benchmark/example.py +++ b/bindings/python/google_benchmark/example.py @@ -13,7 +13,8 @@ # limitations under the License. """Example of Python using C++ benchmark framework. -To run this example, you must first install the `google_benchmark` Python package. +To run this example, you must first install the `google_benchmark` Python +package. To install using `setup.py`, download and extract the `google_benchmark` source. In the extracted directory, execute: @@ -21,6 +22,7 @@ """ import random +import sys import time import google_benchmark as benchmark @@ -38,6 +40,7 @@ def sum_million(state): while state: sum(range(1_000_000)) + @benchmark.register def pause_timing(state): """Pause timing every iteration.""" @@ -56,10 +59,11 @@ def skipped(state): state.skip_with_error("some error") return # NOTE: You must explicitly return, or benchmark will continue. - ... # Benchmark code would be here. + # Benchmark code would be here. @benchmark.register +@benchmark.option.use_manual_time() def manual_timing(state): while state: # Manually count Python CPU time @@ -72,11 +76,10 @@ def manual_timing(state): @benchmark.register def custom_counters(state): - """Collect cutom metric using benchmark.Counter.""" + """Collect custom metric using benchmark.Counter.""" num_foo = 0.0 while state: # Benchmark some code here - pass # Collect some custom metric named foo num_foo += 0.13 @@ -85,7 +88,9 @@ def custom_counters(state): # Set a counter as a rate. state.counters["foo_rate"] = Counter(num_foo, Counter.kIsRate) # Set a counter as an inverse of rate. - state.counters["foo_inv_rate"] = Counter(num_foo, Counter.kIsRate | Counter.kInvert) + state.counters["foo_inv_rate"] = Counter( + num_foo, Counter.kIsRate | Counter.kInvert + ) # Set a counter as a thread-average quantity. state.counters["foo_avg"] = Counter(num_foo, Counter.kAvgThreads) # There's also a combined flag: @@ -133,4 +138,5 @@ def computing_complexity(state): if __name__ == "__main__": + benchmark.add_custom_context("python", sys.version) benchmark.main() diff --git a/bindings/python/pybind11.BUILD b/bindings/python/pybind11.BUILD deleted file mode 100644 index bc83350038..0000000000 --- a/bindings/python/pybind11.BUILD +++ /dev/null @@ -1,20 +0,0 @@ -cc_library( - name = "pybind11", - hdrs = glob( - include = [ - "include/pybind11/*.h", - "include/pybind11/detail/*.h", - ], - exclude = [ - "include/pybind11/common.h", - "include/pybind11/eigen.h", - ], - ), - copts = [ - "-fexceptions", - "-Wno-undefined-inline", - "-Wno-pragma-once-outside-header", - ], - includes = ["include"], - visibility = ["//visibility:public"], -) diff --git a/bindings/python/python_headers.BUILD b/bindings/python/python_headers.BUILD deleted file mode 100644 index 9c34cf6ca4..0000000000 --- a/bindings/python/python_headers.BUILD +++ /dev/null @@ -1,6 +0,0 @@ -cc_library( - name = "python_headers", - hdrs = glob(["**/*.h"]), - includes = ["."], - visibility = ["//visibility:public"], -) diff --git a/bindings/python/requirements.txt b/bindings/python/requirements.txt deleted file mode 100644 index f5bbe7eca5..0000000000 --- a/bindings/python/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -absl-py>=0.7.1 - diff --git a/bindings/rust/Cargo.lock b/bindings/rust/Cargo.lock new file mode 100644 index 0000000000..c0ed48717e --- /dev/null +++ b/bindings/rust/Cargo.lock @@ -0,0 +1,301 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "cc" +version = "1.2.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "codespan-reporting" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" +dependencies = [ + "serde", + "termcolor", + "unicode-width", +] + +[[package]] +name = "cxx" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "747d8437319e3a2f43d93b341c137927ca70c0f5dabeea7a005a73665e247c7e" +dependencies = [ + "cc", + "cxx-build", + "cxxbridge-cmd", + "cxxbridge-flags", + "cxxbridge-macro", + "foldhash", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f4697d190a142477b16aef7da8a99bfdc41e7e8b1687583c0d23a79c7afc1e" +dependencies = [ + "cc", + "codespan-reporting", + "indexmap", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-cmd" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0956799fa8678d4c50eed028f2de1c0552ae183c76e976cf7ca8c4e36a7c328" +dependencies = [ + "clap", + "codespan-reporting", + "indexmap", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23384a836ab4f0ad98ace7e3955ad2de39de42378ab487dc28d3990392cb283a" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6acc6b5822b9526adfb4fc377b67128fdd60aac757cc4a741a6278603f763cf" +dependencies = [ + "indexmap", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "google-benchmark-rs" +version = "1.9.5" +dependencies = [ + "cmake", + "cxx", + "cxx-build", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "link-cplusplus" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82" +dependencies = [ + "cc", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "scratch" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/bindings/rust/Cargo.toml b/bindings/rust/Cargo.toml new file mode 100644 index 0000000000..0a6dda959a --- /dev/null +++ b/bindings/rust/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "google-benchmark-rs" +version = "1.9.5" +edition = "2021" +description = "Rust bindings for google/benchmark" +license = "Apache-2.0" + +[dependencies] +cxx = "1.0" + +[build-dependencies] +cmake = "0.1" +cxx-build = "1.0" diff --git a/bindings/rust/build.rs b/bindings/rust/build.rs new file mode 100644 index 0000000000..02a2f1a14e --- /dev/null +++ b/bindings/rust/build.rs @@ -0,0 +1,40 @@ +fn main() { + let mut config = cmake::Config::new("../../"); + config + .define("BENCHMARK_ENABLE_TESTING", "OFF") + .define("BENCHMARK_ENABLE_LTO", "OFF") + .define("BENCHMARK_ENABLE_WERROR", "OFF") + .build_target("benchmark"); + + // Rust defaults to the Release CRT (/MD) on Windows even in Debug mode. + // Force CMake to use the Release profile so `google/benchmark` uses `/MD`, + // avoiding a mismatch with `cxx_build` (which uses `/MD`). + if cfg!(target_os = "windows") { + config.profile("Release"); + } + + let dst = config.build(); + + println!("cargo:rustc-link-search=native={}/build/src", dst.display()); + println!("cargo:rustc-link-search=native={}/build/src/Debug", dst.display()); + println!("cargo:rustc-link-search=native={}/build/src/Release", dst.display()); + println!("cargo:rustc-link-lib=static=benchmark"); + + cxx_build::bridge("src/ffi.rs") + .file("src/rust_api.cc") + .include("../../include") + .include("src") + .std("c++17") + .define("BENCHMARK_STATIC_DEFINE", None) + .compile("benchmark_rust_ffi"); + + if cfg!(target_os = "windows") { + println!("cargo:rustc-link-lib=shlwapi"); + } + + println!("cargo:rerun-if-changed=src/ffi.rs"); + println!("cargo:rerun-if-changed=src/rust_api.cc"); + println!("cargo:rerun-if-changed=src/rust_api.h"); + println!("cargo:rerun-if-changed=../../src/"); + println!("cargo:rerun-if-changed=../../include/"); +} diff --git a/bindings/rust/src/ffi.rs b/bindings/rust/src/ffi.rs new file mode 100644 index 0000000000..d8ad766d9e --- /dev/null +++ b/bindings/rust/src/ffi.rs @@ -0,0 +1,21 @@ +#[cxx::bridge] +pub mod ffi { + #[namespace = "benchmark"] + unsafe extern "C++" { + include!("benchmark/benchmark.h"); + + type State; + + fn KeepRunning(self: Pin<&mut State>) -> bool; + fn RunSpecifiedBenchmarks() -> usize; + } + + #[namespace = "benchmark::rust_api"] + unsafe extern "C++" { + include!("rust_api.h"); + + unsafe fn SkipWithError(state: Pin<&mut State>, msg: &str); + unsafe fn RegisterBenchmark(name: &str, func: fn(Pin<&mut State>)); + unsafe fn Initialize(argc: *mut i32, argv: usize); + } +} diff --git a/bindings/rust/src/lib.rs b/bindings/rust/src/lib.rs new file mode 100644 index 0000000000..b13c7ecf67 --- /dev/null +++ b/bindings/rust/src/lib.rs @@ -0,0 +1,66 @@ +pub mod ffi; + +use std::ffi::CString; +use std::os::raw::c_char; +use std::pin::Pin; + +pub struct State<'a> { + #[doc(hidden)] + pub inner: Pin<&'a mut ffi::ffi::State>, +} + +impl<'a> State<'a> { + /// Returns true if the benchmark should continue running. + /// + /// **Note:** `keep_running()` currently has a small per-iteration overhead due to the FFI boundary. + /// In the future, this could be optimized using `KeepRunningBatch` under the hood. + #[inline] + pub fn keep_running(&mut self) -> bool { + self.inner.as_mut().KeepRunning() + } + + pub fn skip_with_error(&mut self, msg: &str) { + unsafe { + ffi::ffi::SkipWithError(self.inner.as_mut(), msg); + } + } +} + +/// Initialize the benchmark library. +/// This should be called before `run_specified_benchmarks`. +pub fn initialize(args: &Vec) { + let mut c_args: Vec = args.iter() + .map(|arg| CString::new(arg.as_str()).unwrap()) + .collect(); + + let mut c_ptrs: Vec<*mut c_char> = c_args.iter_mut() + .map(|c| c.as_ptr() as *mut c_char) + .collect(); + + let mut argc = c_ptrs.len() as i32; + let argv = c_ptrs.as_mut_ptr(); + + unsafe { + ffi::ffi::Initialize(&mut argc as *mut _, argv as usize); + } +} + +#[macro_export] +macro_rules! register_benchmark { + ($name:expr, $func:path) => { + { + fn trampoline(mut state: std::pin::Pin<&mut $crate::ffi::ffi::State>) { + let mut wrapped = $crate::State { inner: state.as_mut() }; + $func(&mut wrapped); + } + unsafe { + $crate::ffi::ffi::RegisterBenchmark($name, trampoline); + } + } + }; +} + +/// Run all registered benchmarks. +pub fn run_specified_benchmarks() -> usize { + ffi::ffi::RunSpecifiedBenchmarks() +} diff --git a/bindings/rust/src/rust_api.cc b/bindings/rust/src/rust_api.cc new file mode 100644 index 0000000000..c85997e390 --- /dev/null +++ b/bindings/rust/src/rust_api.cc @@ -0,0 +1,33 @@ +#include "rust_api.h" + +#include + +namespace benchmark { +namespace rust_api { + +void RegisterBenchmark(rust::Str name, rust::Fn func); +void Initialize(int* argc, size_t argv); +void SkipWithError(benchmark::State& state, rust::Str msg); + +void RegisterBenchmark(rust::Str name, rust::Fn func) { + ::benchmark::RegisterBenchmark(std::string(name).c_str(), + [func](benchmark::State& st) { func(st); }); +} + +void Initialize(int* argc, size_t argv) { + char** argv_ptr = reinterpret_cast(argv); + if (argc != nullptr && *argc > 0 && argv_ptr != nullptr && + argv_ptr[0] != nullptr) { + static std::string executable_name; + executable_name = argv_ptr[0]; + argv_ptr[0] = executable_name.data(); + } + ::benchmark::Initialize(argc, argv_ptr); +} + +void SkipWithError(benchmark::State& state, rust::Str msg) { + state.SkipWithError(std::string(msg).c_str()); +} + +} // namespace rust_api +} // namespace benchmark diff --git a/bindings/rust/src/rust_api.h b/bindings/rust/src/rust_api.h new file mode 100644 index 0000000000..17ccd0fd53 --- /dev/null +++ b/bindings/rust/src/rust_api.h @@ -0,0 +1,14 @@ +#pragma once + +#include "benchmark/benchmark.h" +#include "rust/cxx.h" + +namespace benchmark { +namespace rust_api { + +void RegisterBenchmark(rust::Str name, rust::Fn func); +void Initialize(int* argc, size_t argv); +void SkipWithError(benchmark::State& state, rust::Str msg); + +} // namespace rust_api +} // namespace benchmark diff --git a/bindings/rust/tests/test_benchmark.rs b/bindings/rust/tests/test_benchmark.rs new file mode 100644 index 0000000000..025846b654 --- /dev/null +++ b/bindings/rust/tests/test_benchmark.rs @@ -0,0 +1,16 @@ +use google_benchmark_rs::{initialize, register_benchmark, run_specified_benchmarks, State}; + +fn my_benchmark(state: &mut State) { + while state.keep_running() { + // do nothing + } +} + +#[test] +fn test_bindings() { + let args = vec!["--benchmark_format=console".to_string(), "--benchmark_min_time=0.01".to_string()]; + initialize(&args); + register_benchmark!("BM_MyBenchmark", my_benchmark); + let count = run_specified_benchmarks(); + assert!(count > 0); +} diff --git a/cmake/CXXFeatureCheck.cmake b/cmake/CXXFeatureCheck.cmake index 62e6741fe3..59ccddf135 100644 --- a/cmake/CXXFeatureCheck.cmake +++ b/cmake/CXXFeatureCheck.cmake @@ -17,28 +17,35 @@ if(__cxx_feature_check) endif() set(__cxx_feature_check INCLUDED) +option(CXXFEATURECHECK_DEBUG OFF) + function(cxx_feature_check FILE) string(TOLOWER ${FILE} FILE) string(TOUPPER ${FILE} VAR) string(TOUPPER "HAVE_${VAR}" FEATURE) if (DEFINED HAVE_${VAR}) - set(HAVE_${VAR} 1 PARENT_SCOPE) - add_definitions(-DHAVE_${VAR}) + if(HAVE_${VAR}) + add_definitions(-DHAVE_${VAR}) + endif() return() endif() + set(FEATURE_CHECK_CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS}) if (ARGC GREATER 1) message(STATUS "Enabling additional flags: ${ARGV1}") - list(APPEND BENCHMARK_CXX_LINKER_FLAGS ${ARGV1}) + list(APPEND FEATURE_CHECK_CMAKE_FLAGS ${ARGV1}) endif() if (NOT DEFINED COMPILE_${FEATURE}) - message(STATUS "Performing Test ${FEATURE}") if(CMAKE_CROSSCOMPILING) + message(STATUS "Cross-compiling to test ${FEATURE}") try_compile(COMPILE_${FEATURE} ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp - CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS} - LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES}) + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CMAKE_FLAGS ${FEATURE_CHECK_CMAKE_FLAGS} + LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES} + OUTPUT_VARIABLE COMPILE_OUTPUT_VAR) if(COMPILE_${FEATURE}) message(WARNING "If you see build failures due to cross compilation, try setting HAVE_${VAR} to 0") @@ -47,23 +54,32 @@ function(cxx_feature_check FILE) set(RUN_${FEATURE} 1 CACHE INTERNAL "") endif() else() - message(STATUS "Performing Test ${FEATURE}") + message(STATUS "Compiling and running to test ${FEATURE}") try_run(RUN_${FEATURE} COMPILE_${FEATURE} ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp - CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS} - LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES}) + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CMAKE_FLAGS ${FEATURE_CHECK_CMAKE_FLAGS} + LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES} + COMPILE_OUTPUT_VARIABLE COMPILE_OUTPUT_VAR) endif() endif() - if(RUN_${FEATURE} EQUAL 0) - message(STATUS "Performing Test ${FEATURE} -- success") - set(HAVE_${VAR} 1 PARENT_SCOPE) - add_definitions(-DHAVE_${VAR}) - else() - if(NOT COMPILE_${FEATURE}) - message(STATUS "Performing Test ${FEATURE} -- failed to compile") + if(COMPILE_${FEATURE}) + if(DEFINED RUN_${FEATURE} AND RUN_${FEATURE} EQUAL 0) + message(STATUS "Performing Test ${FEATURE} -- success") + set(HAVE_${VAR} 1 CACHE INTERNAL "") + add_definitions(-DHAVE_${VAR}) else() message(STATUS "Performing Test ${FEATURE} -- compiled but failed to run") + set(HAVE_${VAR} 0 CACHE INTERNAL "") + endif() + else() + if(CXXFEATURECHECK_DEBUG) + message(STATUS "Performing Test ${FEATURE} -- failed to compile: ${COMPILE_OUTPUT_VAR}") + else() + message(STATUS "Performing Test ${FEATURE} -- failed to compile") endif() + set(HAVE_${VAR} 0 CACHE INTERNAL "") endif() endfunction() diff --git a/cmake/Config.cmake.in b/cmake/Config.cmake.in index 2e15f0cf82..c65cdb54e3 100644 --- a/cmake/Config.cmake.in +++ b/cmake/Config.cmake.in @@ -4,4 +4,9 @@ include (CMakeFindDependencyMacro) find_dependency (Threads) +if (@BENCHMARK_ENABLE_LIBPFM@) + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") + find_dependency (PFM) +endif() + include("${CMAKE_CURRENT_LIST_DIR}/@targets_export_name@.cmake") diff --git a/cmake/GetGitVersion.cmake b/cmake/GetGitVersion.cmake index 04a1f9b70d..b0210103b2 100644 --- a/cmake/GetGitVersion.cmake +++ b/cmake/GetGitVersion.cmake @@ -20,38 +20,16 @@ set(__get_git_version INCLUDED) function(get_git_version var) if(GIT_EXECUTABLE) - execute_process(COMMAND ${GIT_EXECUTABLE} describe --tags --match "v[0-9]*.[0-9]*.[0-9]*" --abbrev=8 + execute_process(COMMAND ${GIT_EXECUTABLE} describe --tags --match "v[0-9]*.[0-9]*.[0-9]*" --abbrev=8 --dirty WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} RESULT_VARIABLE status - OUTPUT_VARIABLE GIT_DESCRIBE_VERSION + OUTPUT_VARIABLE GIT_VERSION ERROR_QUIET) if(status) - set(GIT_DESCRIBE_VERSION "v0.0.0") + set(GIT_VERSION "v0.0.0") endif() - - string(STRIP ${GIT_DESCRIBE_VERSION} GIT_DESCRIBE_VERSION) - if(GIT_DESCRIBE_VERSION MATCHES v[^-]*-) - string(REGEX REPLACE "v([^-]*)-([0-9]+)-.*" "\\1.\\2" GIT_VERSION ${GIT_DESCRIBE_VERSION}) - else() - string(REGEX REPLACE "v(.*)" "\\1" GIT_VERSION ${GIT_DESCRIBE_VERSION}) - endif() - - # Work out if the repository is dirty - execute_process(COMMAND ${GIT_EXECUTABLE} update-index -q --refresh - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} - OUTPUT_QUIET - ERROR_QUIET) - execute_process(COMMAND ${GIT_EXECUTABLE} diff-index --name-only HEAD -- - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} - OUTPUT_VARIABLE GIT_DIFF_INDEX - ERROR_QUIET) - string(COMPARE NOTEQUAL "${GIT_DIFF_INDEX}" "" GIT_DIRTY) - if (${GIT_DIRTY}) - set(GIT_DESCRIBE_VERSION "${GIT_DESCRIBE_VERSION}-dirty") - endif() - message(STATUS "git version: ${GIT_DESCRIBE_VERSION} normalized to ${GIT_VERSION}") else() - set(GIT_VERSION "0.0.0") + set(GIT_VERSION "v0.0.0") endif() set(${var} ${GIT_VERSION} PARENT_SCOPE) diff --git a/cmake/GoogleTest.cmake b/cmake/GoogleTest.cmake index 44adbfbe4b..e66e9d1a20 100644 --- a/cmake/GoogleTest.cmake +++ b/cmake/GoogleTest.cmake @@ -29,19 +29,25 @@ set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) include(${GOOGLETEST_PREFIX}/googletest-paths.cmake) -# googletest doesn't seem to want to stay build warning clean so let's not hurt ourselves. -if (MSVC) - add_compile_options(/wd4244 /wd4722) -else() - add_compile_options(-w) -endif() - # Add googletest directly to our build. This defines # the gtest and gtest_main targets. add_subdirectory(${GOOGLETEST_SOURCE_DIR} ${GOOGLETEST_BINARY_DIR} EXCLUDE_FROM_ALL) +# googletest doesn't seem to want to stay build warning clean so let's not hurt ourselves. +if (MSVC) + target_compile_options(gtest PRIVATE "/wd4244" "/wd4722") + target_compile_options(gtest_main PRIVATE "/wd4244" "/wd4722") + target_compile_options(gmock PRIVATE "/wd4244" "/wd4722") + target_compile_options(gmock_main PRIVATE "/wd4244" "/wd4722") +else() + target_compile_options(gtest PRIVATE "-w") + target_compile_options(gtest_main PRIVATE "-w") + target_compile_options(gmock PRIVATE "-w") + target_compile_options(gmock_main PRIVATE "-w") +endif() + if(NOT DEFINED GTEST_COMPILE_COMMANDS) set(GTEST_COMPILE_COMMANDS ON) endif() diff --git a/cmake/GoogleTest.cmake.in b/cmake/GoogleTest.cmake.in index ce653ac375..6473892489 100644 --- a/cmake/GoogleTest.cmake.in +++ b/cmake/GoogleTest.cmake.in @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8.12) +cmake_minimum_required (VERSION 3.13...3.22) project(googletest-download NONE) @@ -34,11 +34,12 @@ else() message(SEND_ERROR "Did not find Google Test sources! Either pass correct path in GOOGLETEST_PATH, or enable BENCHMARK_DOWNLOAD_DEPENDENCIES, or disable BENCHMARK_USE_BUNDLED_GTEST, or disable BENCHMARK_ENABLE_GTEST_TESTS / BENCHMARK_ENABLE_TESTING.") return() else() - message(WARNING "Did not find Google Test sources! Fetching from web...") + message(STATUS "Did not find Google Test sources! Fetching from web...") ExternalProject_Add( googletest GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG "release-1.11.0" + GIT_TAG "v1.15.2" + GIT_SHALLOW "ON" PREFIX "${CMAKE_BINARY_DIR}" STAMP_DIR "${CMAKE_BINARY_DIR}/stamp" DOWNLOAD_DIR "${CMAKE_BINARY_DIR}/download" diff --git a/cmake/Modules/FindPFM.cmake b/cmake/Modules/FindPFM.cmake index cf807a1ee9..4c1ce938f9 100644 --- a/cmake/Modules/FindPFM.cmake +++ b/cmake/Modules/FindPFM.cmake @@ -1,26 +1,28 @@ # If successful, the following variables will be defined: -# HAVE_LIBPFM. -# Set BENCHMARK_ENABLE_LIBPFM to 0 to disable, regardless of libpfm presence. -include(CheckIncludeFile) -include(CheckLibraryExists) +# PFM_FOUND. +# PFM_LIBRARIES +# PFM_INCLUDE_DIRS +# the following target will be defined: +# PFM::libpfm + include(FeatureSummary) -enable_language(C) +include(FindPackageHandleStandardArgs) set_package_properties(PFM PROPERTIES URL http://perfmon2.sourceforge.net/ - DESCRIPTION "a helper library to develop monitoring tools" + DESCRIPTION "A helper library to develop monitoring tools" PURPOSE "Used to program specific performance monitoring events") -check_library_exists(libpfm.a pfm_initialize "" HAVE_LIBPFM_INITIALIZE) -if(HAVE_LIBPFM_INITIALIZE) - check_include_file(perfmon/perf_event.h HAVE_PERFMON_PERF_EVENT_H) - check_include_file(perfmon/pfmlib.h HAVE_PERFMON_PFMLIB_H) - check_include_file(perfmon/pfmlib_perf_event.h HAVE_PERFMON_PFMLIB_PERF_EVENT_H) - if(HAVE_PERFMON_PERF_EVENT_H AND HAVE_PERFMON_PFMLIB_H AND HAVE_PERFMON_PFMLIB_PERF_EVENT_H) - message("Using Perf Counters.") - set(HAVE_LIBPFM 1) - set(PFM_FOUND 1) - endif() -else() - message("Perf Counters support requested, but was unable to find libpfm.") +find_library(PFM_LIBRARY NAMES pfm) +find_path(PFM_INCLUDE_DIR NAMES perfmon/pfmlib.h) + +find_package_handle_standard_args(PFM REQUIRED_VARS PFM_LIBRARY PFM_INCLUDE_DIR) + +if (PFM_FOUND AND NOT TARGET PFM::libpfm) + add_library(PFM::libpfm UNKNOWN IMPORTED) + set_target_properties(PFM::libpfm PROPERTIES + IMPORTED_LOCATION "${PFM_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${PFM_INCLUDE_DIR}") endif() + +mark_as_advanced(PFM_LIBRARY PFM_INCLUDE_DIR) diff --git a/cmake/benchmark.pc.in b/cmake/benchmark.pc.in index 34beb012ee..520d14971f 100644 --- a/cmake/benchmark.pc.in +++ b/cmake/benchmark.pc.in @@ -1,12 +1,12 @@ prefix=@CMAKE_INSTALL_PREFIX@ exec_prefix=${prefix} -libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ -includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +libdir=@BENCHMARK_PKG_CONFIG_LIBDIR@ +includedir=@BENCHMARK_PKG_CONFIG_INCLUDEDIR@ Name: @PROJECT_NAME@ Description: Google microbenchmark framework -Version: @VERSION@ +Version: @NORMALIZED_VERSION@ Libs: -L${libdir} -lbenchmark -Libs.private: -lpthread +Libs.private: -lpthread @BENCHMARK_PRIVATE_LINK_LIBRARIES@ Cflags: -I${includedir} diff --git a/cmake/benchmark_main.pc.in b/cmake/benchmark_main.pc.in new file mode 100644 index 0000000000..249de6cf98 --- /dev/null +++ b/cmake/benchmark_main.pc.in @@ -0,0 +1,9 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=@BENCHMARK_PKG_CONFIG_LIBDIR@ + +Name: @PROJECT_NAME@ +Description: Google microbenchmark framework (with main() function) +Version: @NORMALIZED_VERSION@ +Requires: benchmark +Libs: -L${libdir} -lbenchmark_main diff --git a/cmake/pthread_affinity.cpp b/cmake/pthread_affinity.cpp new file mode 100644 index 0000000000..2eea573130 --- /dev/null +++ b/cmake/pthread_affinity.cpp @@ -0,0 +1,19 @@ +#include +#ifdef __FreeBSD__ +#include +#endif +int main() { + cpu_set_t set; + CPU_ZERO(&set); + for (int i = 0; i < CPU_SETSIZE; ++i) { + CPU_SET(i, &set); + CPU_CLR(i, &set); + } + pthread_t self = pthread_self(); + int ret; + ret = pthread_getaffinity_np(self, sizeof(set), &set); + if (ret != 0) return ret; + ret = pthread_setaffinity_np(self, sizeof(set), &set); + if (ret != 0) return ret; + return 0; +} diff --git a/cmake/thread_safety_attributes.cpp b/cmake/thread_safety_attributes.cpp index 46161babdb..1069a90a21 100644 --- a/cmake/thread_safety_attributes.cpp +++ b/cmake/thread_safety_attributes.cpp @@ -1,4 +1,25 @@ -#define HAVE_THREAD_SAFETY_ATTRIBUTES -#include "../src/mutex.h" +#if defined(__clang__) +#define THREAD_ANNOTATION_ATTRIBUTE_(x) __attribute__((x)) +#else +#define THREAD_ANNOTATION_ATTRIBUTE_(x) +#endif -int main() {} +#define CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE_(capability(x)) +#define ACQUIRE() THREAD_ANNOTATION_ATTRIBUTE_(acquire_capability()) +#define RELEASE() THREAD_ANNOTATION_ATTRIBUTE_(release_capability()) + +class CAPABILITY("mutex") Mutex { + public: + void lock() ACQUIRE(); + void unlock() RELEASE(); +}; + +void Mutex::lock() ACQUIRE() {} +void Mutex::unlock() RELEASE() {} + +int main() { + Mutex m; + m.lock(); + m.unlock(); + return 0; +} diff --git a/config/generate_export_header.bzl b/config/generate_export_header.bzl deleted file mode 100644 index bf98092d59..0000000000 --- a/config/generate_export_header.bzl +++ /dev/null @@ -1,168 +0,0 @@ -# -# Original file is located at: -# https://github.com/RobotLocomotion/drake/blob/bad032aeb09b13c7f8c87ed64b624c8d1e9adb30/tools/workspace/generate_export_header.bzl -# -# All components of Drake are licensed under the BSD 3-Clause License -# shown below. Where noted in the source code, some portions may -# be subject to other permissive, non-viral licenses. -# -# Copyright 2012-2016 Robot Locomotion Group @ CSAIL -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. Redistributions -# in binary form must reproduce the above copyright notice, this list of -# conditions and the following disclaimer in the documentation and/or -# other materials provided with the distribution. Neither the name of -# the Massachusetts Institute of Technology nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# -*- python -*- - -# Defines the implementation actions to generate_export_header. -def _generate_export_header_impl(ctx): - windows_constraint = ctx.attr._windows_constraint[platform_common.ConstraintValueInfo] - output = ctx.outputs.out - - if ctx.target_platform_has_constraint(windows_constraint): - export_attr = "__declspec(dllexport)" - import_attr = "__declspec(dllimport)" - no_export_attr = "" - deprecated_attr = "__declspec(deprecated)" - else: - export_attr = "__attribute__((visibility(\"default\")))" - import_attr = "__attribute__((visibility(\"default\")))" - no_export_attr = "__attribute__((visibility(\"hidden\")))" - deprecated_attr = "__attribute__((__deprecated__))" - - content = [ - "#ifndef %s_H" % ctx.attr.export_macro_name, - "#define %s_H" % ctx.attr.export_macro_name, - "", - "#ifdef %s" % ctx.attr.static_define, - "# define %s" % ctx.attr.export_macro_name, - "# define %s" % ctx.attr.no_export_macro_name, - "#else", - "# ifndef %s" % ctx.attr.export_macro_name, - "# ifdef %s" % ctx.attr.export_import_condition, - "# define %s %s" % (ctx.attr.export_macro_name, export_attr), - "# else", - "# define %s %s" % (ctx.attr.export_macro_name, import_attr), - "# endif", - "# endif", - "# ifndef %s" % ctx.attr.no_export_macro_name, - "# define %s %s" % (ctx.attr.no_export_macro_name, no_export_attr), - "# endif", - "#endif", - "", - "#ifndef %s" % ctx.attr.deprecated_macro_name, - "# define %s %s" % (ctx.attr.deprecated_macro_name, deprecated_attr), - "#endif", - "", - "#ifndef %s" % ctx.attr.export_deprecated_macro_name, - "# define %s %s %s" % (ctx.attr.export_deprecated_macro_name, ctx.attr.export_macro_name, ctx.attr.deprecated_macro_name), # noqa - "#endif", - "", - "#ifndef %s" % ctx.attr.no_export_deprecated_macro_name, - "# define %s %s %s" % (ctx.attr.no_export_deprecated_macro_name, ctx.attr.no_export_macro_name, ctx.attr.deprecated_macro_name), # noqa - "#endif", - "", - "#endif", - ] - - ctx.actions.write(output = output, content = "\n".join(content) + "\n") - -# Defines the rule to generate_export_header. -_generate_export_header_gen = rule( - attrs = { - "out": attr.output(mandatory = True), - "export_import_condition": attr.string(), - "export_macro_name": attr.string(), - "deprecated_macro_name": attr.string(), - "export_deprecated_macro_name": attr.string(), - "no_export_macro_name": attr.string(), - "no_export_deprecated_macro_name": attr.string(), - "static_define": attr.string(), - "_windows_constraint": attr.label(default = "@platforms//os:windows"), - }, - output_to_genfiles = True, - implementation = _generate_export_header_impl, -) - -def generate_export_header( - lib = None, - name = None, - out = None, - export_import_condition = None, - export_macro_name = None, - deprecated_macro_name = None, - export_deprecated_macro_name = None, - no_export_macro_name = None, - no_export_deprecated_macro_name = None, - static_define = None, - **kwargs): - """ - Creates a rule to generate an export header for a named library. - - This is an incomplete implementation of CMake's generate_export_header. (In - particular, it assumes a platform that uses - __attribute__((visibility("default"))) to decorate exports.) - - By default, the rule will have a mangled name related to the library name, - and will produce "_export.h". - - The CMake documentation of the generate_export_header macro is: - https://cmake.org/cmake/help/latest/module/GenerateExportHeader.html - - """ - - if name == None: - name = "__%s_export_h" % lib - if out == None: - out = "%s_export.h" % lib - if export_import_condition == None: - # CMake does not uppercase the _EXPORTS define. - export_import_condition = "%s_EXPORTS" % lib - if export_macro_name == None: - export_macro_name = "%s_EXPORT" % lib.upper() - if deprecated_macro_name == None: - deprecated_macro_name = "%s_DEPRECATED" % lib.upper() - if export_deprecated_macro_name == None: - export_deprecated_macro_name = "%s_DEPRECATED_EXPORT" % lib.upper() - if no_export_macro_name == None: - no_export_macro_name = "%s_NO_EXPORT" % lib.upper() - if no_export_deprecated_macro_name == None: - no_export_deprecated_macro_name = \ - "%s_DEPRECATED_NO_EXPORT" % lib.upper() - if static_define == None: - static_define = "%s_STATIC_DEFINE" % lib.upper() - - _generate_export_header_gen( - name = name, - out = out, - export_import_condition = export_import_condition, - export_macro_name = export_macro_name, - deprecated_macro_name = deprecated_macro_name, - export_deprecated_macro_name = export_deprecated_macro_name, - no_export_macro_name = no_export_macro_name, - no_export_deprecated_macro_name = no_export_deprecated_macro_name, - static_define = static_define, - **kwargs - ) diff --git a/docs/AssemblyTests.md b/docs/AssemblyTests.md index 1fbdc269b5..89df7ca520 100644 --- a/docs/AssemblyTests.md +++ b/docs/AssemblyTests.md @@ -111,6 +111,7 @@ between compilers or compiler versions. A common example of this is matching stack frame addresses. In this case regular expressions can be used to match the differing bits of output. For example: + ```c++ int ExternInt; struct Point { int x, y, z; }; @@ -127,6 +128,7 @@ extern "C" void test_store_point() { // CHECK: ret } ``` + ## Current Requirements and Limitations diff --git a/docs/_config.yml b/docs/_config.yml index 2f7efbeab5..32f9f2e0dd 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -1 +1,3 @@ -theme: jekyll-theme-minimal \ No newline at end of file +theme: jekyll-theme-minimal +logo: /assets/images/icon_black.png +show_downloads: true diff --git a/docs/assets/images/icon.png b/docs/assets/images/icon.png new file mode 100644 index 0000000000..b98260486e Binary files /dev/null and b/docs/assets/images/icon.png differ diff --git a/docs/assets/images/icon.xcf b/docs/assets/images/icon.xcf new file mode 100644 index 0000000000..f2f0be415f Binary files /dev/null and b/docs/assets/images/icon.xcf differ diff --git a/docs/assets/images/icon_black.png b/docs/assets/images/icon_black.png new file mode 100644 index 0000000000..656ae797cf Binary files /dev/null and b/docs/assets/images/icon_black.png differ diff --git a/docs/assets/images/icon_black.xcf b/docs/assets/images/icon_black.xcf new file mode 100644 index 0000000000..430e7bafe5 Binary files /dev/null and b/docs/assets/images/icon_black.xcf differ diff --git a/docs/bazel.md b/docs/bazel.md new file mode 100644 index 0000000000..b5217a97e8 --- /dev/null +++ b/docs/bazel.md @@ -0,0 +1,92 @@ +# Bazel + +Google Benchmark provides Bazel targets for both the benchmark library and the +optional default `main` function: + +* `@google_benchmark//:benchmark` provides the benchmark library. +* `@google_benchmark//:benchmark_main` provides the default `main` function and + depends on `@google_benchmark//:benchmark`. + +Use `@google_benchmark//:benchmark` when the benchmark target defines its own +`main` function, including through `BENCHMARK_MAIN()`. Use +`@google_benchmark//:benchmark_main` when the benchmark target should use the +default Google Benchmark entry point. + +## Bzlmod + +With Bzlmod enabled, add Google Benchmark to your `MODULE.bazel` file: + +```starlark +bazel_dep(name = "google_benchmark", version = "") +``` + +Replace `` with the Google Benchmark release version you want to use. + +Then depend on the Bazel target from a `cc_binary` or `cc_test`: + +```starlark +load("@rules_cc//cc:defs.bzl", "cc_binary") + +cc_binary( + name = "string_benchmark", + srcs = ["string_benchmark.cc"], + deps = ["@google_benchmark//:benchmark_main"], +) +``` + +The source file should register benchmarks, but it should not call +`BENCHMARK_MAIN()` when linking against `@google_benchmark//:benchmark_main`: + +```c++ +#include +#include + +static void BM_StringCreation(benchmark::State& state) { + for (auto _ : state) { + std::string empty_string; + } +} +BENCHMARK(BM_StringCreation); +``` + +Run the benchmark with Bazel: + +```bash +bazel run //:string_benchmark +``` + +Pass Google Benchmark flags after Bazel's `--` separator: + +```bash +bazel run //:string_benchmark -- --benchmark_filter=StringCreation +``` + +## WORKSPACE + +Projects that still use `WORKSPACE` can declare Google Benchmark as an external +repository and load its dependencies from `bazel/benchmark_deps.bzl`: + +```starlark +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "google_benchmark", + strip_prefix = "benchmark-", + urls = ["https://github.com/google/benchmark/archive/refs/tags/v.tar.gz"], + # Add sha256 for reproducible builds. +) + +load("@google_benchmark//:bazel/benchmark_deps.bzl", "benchmark_deps") + +benchmark_deps() +``` + +Use the same `` value without the leading `v`; the archive URL adds the tag prefix explicitly. + +After declaring the repository, use the same target labels shown above: +`@google_benchmark//:benchmark` or `@google_benchmark//:benchmark_main`. + +## Perf Counters + +When using Bazel, enable libpfm support by adding `--define pfm=1` to the build +or run command. See [Perf Counters](perf_counters.md) for more details. diff --git a/docs/dependencies.md b/docs/dependencies.md index 7af52b95bd..fdb7f19b2f 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -1,19 +1,19 @@ # Build tool dependency policy -To ensure the broadest compatibility when building the benchmark library, but -still allow forward progress, we require any build tooling to be available for: +We follow the [Foundational C++ support policy](https://opensource.google/documentation/policies/cplusplus-support) for our build tools. In +particular the ["Build Systems" section](https://opensource.google/documentation/policies/cplusplus-support#build-systems). -* Debian stable _and_ -* The last two Ubuntu LTS releases +## CMake -Currently, this means using build tool versions that are available for Ubuntu -18.04 (Bionic Beaver), Ubuntu 20.04 (Focal Fossa), and Debian 11 (bullseye). +The current supported version is CMake 3.13 as of 2024-10-24. Most modern +distributions include newer versions, for example: -_Note, CI also runs ubuntu-16.04 and ubuntu-14.04 to ensure best effort support -for older versions._ +* Ubuntu 20.04 provides CMake 3.16.3 +* Debian 11.4 provides CMake 3.18.4 +* Ubuntu 22.04 provides CMake 3.22.1 -## cmake -The current supported version is cmake 3.5.1 as of 2018-06-06. +## Python -_Note, this version is also available for Ubuntu 14.04, an older Ubuntu LTS -release, as `cmake3`._ +The Python bindings require Python 3.10+ as of v1.9.0 (2024-08-16) for installation from PyPI. +Building from source for older versions probably still works, though. See the [user guide](python_bindings.md) for details on how to build from source. +The minimum theoretically supported version is Python 3.8, since the used bindings generator (nanobind) only supports Python 3.8+. diff --git a/docs/index.md b/docs/index.md index eb82eff9ee..af6e88aea1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,10 +1,13 @@ # Benchmark * [Assembly Tests](AssemblyTests.md) +* [Bazel](bazel.md) * [Dependencies](dependencies.md) * [Perf Counters](perf_counters.md) * [Platform Specific Build Instructions](platform_specific_build_instructions.md) +* [Python Bindings](python_bindings.md) * [Random Interleaving](random_interleaving.md) +* [Reducing Variance](reducing_variance.md) * [Releasing](releasing.md) * [Tools](tools.md) -* [User Guide](user_guide.md) \ No newline at end of file +* [User Guide](user_guide.md) diff --git a/docs/perf_counters.md b/docs/perf_counters.md index 74560e9669..f342092c99 100644 --- a/docs/perf_counters.md +++ b/docs/perf_counters.md @@ -12,16 +12,17 @@ This feature is available if: * The benchmark is run on an architecture featuring a Performance Monitoring Unit (PMU), * The benchmark is compiled with support for collecting counters. Currently, - this requires [libpfm](http://perfmon2.sourceforge.net/) be available at build - time + this requires [libpfm](http://perfmon2.sourceforge.net/), which is built as a + dependency via Bazel. The feature does not require modifying benchmark code. Counter collection is handled at the boundaries where timer collection is also handled. To opt-in: - -* Install `libpfm4-dev`, e.g. `apt-get install libpfm4-dev`. -* Enable the cmake flag BENCHMARK_ENABLE_LIBPFM. +* If using a Bazel build, add `--define pfm=1` to your build flags +* If using CMake: + * Install `libpfm4-dev`, e.g. `apt-get install libpfm4-dev`. + * Enable the CMake flag `BENCHMARK_ENABLE_LIBPFM` in `CMakeLists.txt`. To use, pass a comma-separated list of counter names through the `--benchmark_perf_counters` flag. The names are decoded through libpfm - meaning, @@ -31,4 +32,4 @@ mapped by libpfm to platform-specifics - see libpfm The counter values are reported back through the [User Counters](../README.md#custom-counters) mechanism, meaning, they are available in all the formats (e.g. JSON) supported -by User Counters. \ No newline at end of file +by User Counters. diff --git a/docs/platform_specific_build_instructions.md b/docs/platform_specific_build_instructions.md index 2d5d6c47ee..5c1439d0a9 100644 --- a/docs/platform_specific_build_instructions.md +++ b/docs/platform_specific_build_instructions.md @@ -15,22 +15,26 @@ On QNX, the pthread library is part of libc and usually included automatically [`pthread_create()`](https://www.qnx.com/developers/docs/7.1/index.html#com.qnx.doc.neutrino.lib_ref/topic/p/pthread_create.html)). There's no separate pthread library to link. -## Building with Visual Studio 2015 or 2017 +## Building with Visual Studio 2015, 2017 or 2022 The `shlwapi` library (`-lshlwapi`) is required to support a call to `CPUInfo` which reads the registry. Either add `shlwapi.lib` under `[ Configuration Properties > Linker > Input ]`, or use the following: ``` // Alternatively, can add libraries using linker options. + +// First, Add the path to the generated library files (directory containing the `benchmark.lib`) in `[Configuration Properties > Linker > General > Additional Library Directories]`. Then do the following: #ifdef _WIN32 #pragma comment ( lib, "Shlwapi.lib" ) #ifdef _DEBUG -#pragma comment ( lib, "benchmarkd.lib" ) +#pragma comment ( lib, "benchmark.lib" ) #else #pragma comment ( lib, "benchmark.lib" ) #endif #endif ``` +When using the static library, make sure to add `BENCHMARK_STATIC_DEFINE` under `[Configuration Properties > C/C++ > Preprocessor > Preprocessor Definitions]` + Can also use the graphical version of CMake: * Open `CMake GUI`. * Under `Where to build the binaries`, same path as source plus `build`. diff --git a/docs/python_bindings.md b/docs/python_bindings.md index 6a7aab0a29..d9c5d2d3f6 100644 --- a/docs/python_bindings.md +++ b/docs/python_bindings.md @@ -3,7 +3,7 @@ Python bindings are available as wheels on [PyPI](https://pypi.org/project/google-benchmark/) for importing and using Google Benchmark directly in Python. Currently, pre-built wheels exist for macOS (both ARM64 and Intel x86), Linux x86-64 and 64-bit Windows. -Supported Python versions are Python 3.7 - 3.10. +Supported Python versions are Python 3.8 - 3.12. To install Google Benchmark's Python bindings, run: @@ -25,9 +25,9 @@ python3 -m venv venv --system-site-packages source venv/bin/activate # .\venv\Scripts\Activate.ps1 on Windows # upgrade Python's system-wide packages -python -m pip install --upgrade pip setuptools wheel -# builds the wheel and stores it in the directory "wheelhouse". -python -m pip wheel . -w wheelhouse +python -m pip install --upgrade pip build +# builds the wheel and stores it in the directory "dist". +python -m build ``` NB: Building wheels from source requires Bazel. For platform-specific instructions on how to install Bazel, diff --git a/docs/reducing_variance.md b/docs/reducing_variance.md new file mode 100644 index 0000000000..364f4af15b --- /dev/null +++ b/docs/reducing_variance.md @@ -0,0 +1,138 @@ +# Reducing Variance + + + +## Disabling CPU Frequency Scaling + +If you see this error: + +``` +***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead. +``` + +you might want to disable the CPU frequency scaling while running the +benchmark, as well as consider other ways to stabilize the performance of +your system while benchmarking. + +Exactly how to do this depends on the Linux distribution, +desktop environment, and installed programs. Specific details are a moving +target, so we will not attempt to exhaustively document them here. + +One simple option is to use the `cpupower` program to change the +performance governor to "performance". This tool is maintained along with +the Linux kernel and provided by your distribution. + +It must be run as root, like this: + +```bash +sudo cpupower frequency-set --governor performance +``` + +After this you can verify that all CPUs are using the performance governor +by running this command: + +```bash +cpupower frequency-info -o proc +``` + +The benchmarks you subsequently run will have less variance. + + + +## Disabling ASLR + +If you see this error: + +``` +***WARNING*** ASLR is enabled, the results may have unreproducible noise in them. +``` + +you might want to disable the ASLR security hardening feature while running the +benchmark. + +The simplest way is to add +``` +benchmark::MaybeReenterWithoutASLR(argc, argv); +``` +as the first line of your `main()` function. It will try to disable ASLR +for the current processor, and, if successful, re-execute the binary. +Note that `personality(2)` may be forbidden by e.g. seccomp (which happens +by default if you are running in a Docker container). + +Note that if you link to `benchmark_main` already does that for you. + +To globally disable ASLR on Linux, run +``` +echo 0 > /proc/sys/kernel/randomize_va_space +``` + +To run a single benchmark with ASLR disabled on Linux, do: +``` +setarch `uname -m` -R ./a_benchmark +``` + +Note that for the information on how to disable ASLR on other operating systems, +please refer to their documentation. + +## Reducing Variance in Benchmarks + +The Linux CPU frequency governor [discussed +above](user_guide#disabling-cpu-frequency-scaling) is not the only source +of noise in benchmarks. Some, but not all, of the sources of variance +include: + +1. On multi-core machines not all CPUs/CPU cores/CPU threads run the same + speed, so running a benchmark one time and then again may give a + different result depending on which CPU it ran on. +2. CPU scaling features that run on the CPU, like Intel's Turbo Boost and + AMD Turbo Core and Precision Boost, can temporarily change the CPU + frequency even when the using the "performance" governor on Linux. +3. Context switching between CPUs, or scheduling competition on the CPU the + benchmark is running on. +4. Intel Hyperthreading or AMD SMT causing the same issue as above. +5. Cache effects caused by code running on other CPUs. +6. Non-uniform memory architectures (NUMA). + +These can cause variance in benchmarks results within a single run +(`--benchmark_repetitions=N`) or across multiple runs of the benchmark +program. + +Reducing sources of variance is OS and architecture dependent, which is one +reason some companies maintain machines dedicated to performance testing. + +Some of the easier and effective ways of reducing variance on a typical +Linux workstation are: + +1. Use the performance governor as [discussed +above](user_guide#disabling-cpu-frequency-scaling). +2. Disable processor boosting by: + ```sh + echo 0 | sudo tee /sys/devices/system/cpu/cpufreq/boost + ``` + See the Linux kernel's + [boost.txt](https://www.kernel.org/doc/Documentation/cpu-freq/boost.txt) + for more information. +3. Set the benchmark program's task affinity to a fixed cpu. For example: + ```sh + taskset -c 0 ./mybenchmark + ``` +4. Increase the program's scheduling priority to minimize context switches using `nice` or `chrt`: + ```sh + sudo nice -n -20 ./mybenchmark + sudo chrt -f 80 ./mybenchmark + ``` +5. Disabling Hyperthreading/SMT. This can be done in the Bios or using the + `/sys` file system (see the LLVM project's [Benchmarking + tips](https://llvm.org/docs/Benchmarking.html)). +6. Close other programs that do non-trivial things based on timers, such as + your web browser, desktop environment, etc. +7. Reduce the working set of your benchmark to fit within the L1 cache, but + do be aware that this may lead you to optimize for an unrealistic + situation. + +Further resources on this topic: + +1. The LLVM project's [Benchmarking + tips](https://llvm.org/docs/Benchmarking.html). +1. The Arch Wiki [Cpu frequency +scaling](https://wiki.archlinux.org/title/CPU_frequency_scaling) page. diff --git a/docs/releasing.md b/docs/releasing.md index 6d3b6138cc..a58dacd8e5 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -8,23 +8,29 @@ * `git log $(git describe --abbrev=0 --tags)..HEAD` gives you the list of commits between the last annotated tag and HEAD * Pick the most interesting. -* Create one last commit that updates the version saved in `CMakeLists.txt` and the - `__version__` variable in `bindings/python/google_benchmark/__init__.py`to the release - version you're creating. (This version will be used if benchmark is installed from the - archive you'll be creating in the next step.) +* Create one last commit that updates the version saved in `CMakeLists.txt`, `MODULE.bazel`, + `bindings/python/google_benchmark/__init__.py`, and `bindings/rust/Cargo.toml` to the release version you're creating. + (This version will be used if benchmark is installed from the archive you'll be creating + in the next step.) ``` -project (benchmark VERSION 1.6.0 LANGUAGES CXX) +# CMakeLists.txt +project (benchmark VERSION 1.9.0 LANGUAGES CXX) ``` -```python -# bindings/python/google_benchmark/__init__.py - -# ... +``` +# MODULE.bazel +module(name = "com_github_google_benchmark", version="1.9.0") +``` -__version__ = "1.6.0" # <-- change this to the release version you are creating +``` +# google_benchmark/__init__.py +__version__ = "1.9.0" +``` -# ... +```toml +# bindings/rust/Cargo.toml +version = "1.9.0" ``` * Create a release through github's interface @@ -34,4 +40,4 @@ __version__ = "1.6.0" # <-- change this to the release version you are creating * `git tag -a -f ` * `git push --force --tags origin` * Confirm that the "Build and upload Python wheels" action runs to completion - * run it manually if it hasn't run + * Run it manually if it hasn't run. diff --git a/docs/tools.md b/docs/tools.md index f2d0c497f3..411f41d405 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -186,6 +186,146 @@ Benchmark Time CPU Time Old This is a mix of the previous two modes, two (potentially different) benchmark binaries are run, and a different filter is applied to each one. As you can note, the values in `Time` and `CPU` columns are calculated as `(new - old) / |old|`. +### Note: Interpreting the output + +Performance measurements are an art, and performance comparisons are doubly so. +Results are often noisy and don't necessarily have large absolute differences to +them, so just by visual inspection, it is not at all apparent if two +measurements are actually showing a performance change or not. It is even more +confusing with multiple benchmark repetitions. + +Thankfully, what we can do, is use statistical tests on the results to determine +whether the performance has statistically-significantly changed. `compare.py` +uses [Mann–Whitney U +test](https://en.wikipedia.org/wiki/Mann%E2%80%93Whitney_U_test), with a null +hypothesis being that there's no difference in performance. + +**The below output is a summary of a benchmark comparison with statistics +provided for a multi-threaded process.** +``` +Benchmark Time CPU Time Old Time New CPU Old CPU New +----------------------------------------------------------------------------------------------------------------------------- +benchmark/threads:1/process_time/real_time_pvalue 0.0000 0.0000 U Test, Repetitions: 27 vs 27 +benchmark/threads:1/process_time/real_time_mean -0.1442 -0.1442 90 77 90 77 +benchmark/threads:1/process_time/real_time_median -0.1444 -0.1444 90 77 90 77 +benchmark/threads:1/process_time/real_time_stddev +0.3974 +0.3933 0 0 0 0 +benchmark/threads:1/process_time/real_time_cv +0.6329 +0.6280 0 0 0 0 +OVERALL_GEOMEAN -0.1442 -0.1442 0 0 0 0 +``` +-------------------------------------------- +Here's a breakdown of each row: + +**benchmark/threads:1/process_time/real_time_pvalue**: This shows the _p-value_ for +the statistical test comparing the performance of the process running with one +thread. A value of 0.0000 suggests a statistically significant difference in +performance. The comparison was conducted using the U Test (Mann-Whitney +U Test) with 27 repetitions for each case. + +**benchmark/threads:1/process_time/real_time_mean**: This shows the relative +difference in mean execution time between two different cases. The negative +value (-0.1442) implies that the new process is faster by about 14.42%. The old +time was 90 units, while the new time is 77 units. + +**benchmark/threads:1/process_time/real_time_median**: Similarly, this shows the +relative difference in the median execution time. Again, the new process is +faster by 14.44%. + +**benchmark/threads:1/process_time/real_time_stddev**: This is the relative +difference in the standard deviation of the execution time, which is a measure +of how much variation or dispersion there is from the mean. A positive value +(+0.3974) implies there is more variance in the execution time in the new +process. + +**benchmark/threads:1/process_time/real_time_cv**: CV stands for Coefficient of +Variation. It is the ratio of the standard deviation to the mean. It provides a +standardized measure of dispersion. An increase (+0.6329) indicates more +relative variability in the new process. + +**OVERALL_GEOMEAN**: Geomean stands for geometric mean, a type of average that is +less influenced by outliers. The negative value indicates a general improvement +in the new process. However, given the values are all zero for the old and new +times, this seems to be a mistake or placeholder in the output. + +----------------------------------------- + + + +Let's first try to see what the different columns represent in the above +`compare.py` benchmarking output: + + 1. **Benchmark:** The name of the function being benchmarked, along with the + size of the input (after the slash). + + 2. **Time:** The average time per operation, across all iterations. + + 3. **CPU:** The average CPU time per operation, across all iterations. + + 4. **Iterations:** The number of iterations the benchmark was run to get a + stable estimate. + + 5. **Time Old and Time New:** These represent the average time it takes for a + function to run in two different scenarios or versions. For example, you + might be comparing how fast a function runs before and after you make some + changes to it. + + 6. **CPU Old and CPU New:** These show the average amount of CPU time that the + function uses in two different scenarios or versions. This is similar to + Time Old and Time New, but focuses on CPU usage instead of overall time. + +In the comparison section, the relative differences in both time and CPU time +are displayed for each input size. + + +A statistically-significant difference is determined by a **p-value**, which is +a measure of the probability that the observed difference could have occurred +just by random chance. A smaller p-value indicates stronger evidence against the +null hypothesis. + +**Therefore:** + 1. If the p-value is less than the chosen significance level (alpha), we + reject the null hypothesis and conclude the benchmarks are significantly + different. + 2. If the p-value is greater than or equal to alpha, we fail to reject the + null hypothesis and treat the two benchmarks as similar. + + + +The result of said the statistical test is additionally communicated through color coding: +```diff ++ Green: +``` + The benchmarks are _**statistically different**_. This could mean the + performance has either **significantly improved** or **significantly + deteriorated**. You should look at the actual performance numbers to see which + is the case. +```diff +- Red: +``` + The benchmarks are _**statistically similar**_. This means the performance + **hasn't significantly changed**. + +In statistical terms, **'green'** means we reject the null hypothesis that +there's no difference in performance, and **'red'** means we fail to reject the +null hypothesis. This might seem counter-intuitive if you're expecting 'green' +to mean 'improved performance' and 'red' to mean 'worsened performance'. +```bash + But remember, in this context: + + 'Success' means 'successfully finding a difference'. + 'Failure' means 'failing to find a difference'. +``` + + +Also, please note that **even if** we determine that there **is** a +statistically-significant difference between the two measurements, it does not +_necessarily_ mean that the actual benchmarks that were measured **are** +different, or vice versa, even if we determine that there is **no** +statistically-significant difference between the two measurements, it does not +necessarily mean that the actual benchmarks that were measured **are not** +different. + + + ### U test If there is a sufficient repetition count of the benchmarks, the tool can do diff --git a/docs/user_guide.md b/docs/user_guide.md index f96b7dfd03..3fed9261d7 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -2,6 +2,8 @@ ## Command Line +[Command Line Options](#command-line-options) + [Output Formats](#output-formats) [Output Files](#output-files) @@ -28,6 +30,8 @@ [Templated Benchmarks](#templated-benchmarks) +[Templated Benchmarks that take arguments](#templated-benchmarks-with-arguments) + [Fixtures](#fixtures) [Custom Counters](#custom-counters) @@ -56,10 +60,216 @@ [Exiting with an Error](#exiting-with-an-error) -[A Faster KeepRunning Loop](#a-faster-keep-running-loop) +[A Faster `KeepRunning` Loop](#a-faster-keep-running-loop) + +## Benchmarking Tips [Disabling CPU Frequency Scaling](#disabling-cpu-frequency-scaling) +[Reducing Variance in Benchmarks](reducing_variance.md) + + +## Command Line Options + +Benchmarks accept options that may be specified either through their command line interface or by setting environment variables before execution. For every `--option_flag=` CLI switch, a corresponding environment variable `OPTION_FLAG=` exists and is used as default if set (CLI switches always prevail). + +### Benchmark Selection and Execution + +#### `--benchmark_list_tests` (BENCHMARK_LIST_TESTS) + +Print a list of all benchmark names and exit. This option overrides all other options. + +**Example:** +```bash +$ ./benchmark --benchmark_list_tests +BM_SomeFunction +BM_AnotherFunction +``` + +#### `--benchmark_filter=` (BENCHMARK_FILTER) + +A regular expression that specifies the set of benchmarks to execute. If this flag is empty, or if this flag is the string "all", all benchmarks linked into the binary are run. + +**Example:** +```bash +$ ./benchmark --benchmark_filter=BM_memcpy/32 +``` + +#### `--benchmark_dry_run` (BENCHMARK_DRY_RUN) + +If enabled, forces each benchmark to execute exactly one iteration and one repetition, bypassing any configured `MinTime()`, `MinWarmUpTime()`, `Iterations()`, or `Repetitions()`. This is useful for quickly verifying that benchmarks can run successfully without waiting for full execution. + +**Example:** +```bash +$ ./benchmark --benchmark_dry_run +``` + +#### `--benchmark_enable_random_interleaving` (BENCHMARK_ENABLE_RANDOM_INTERLEAVING) + +If set, enable random interleaving of repetitions of all benchmarks. This can help reduce the impact of system state changes on benchmark results. See [GitHub issue #1051](https://github.com/google/benchmark/issues/1051) for details. + +**Example:** +```bash +$ ./benchmark --benchmark_enable_random_interleaving +``` + +### Timing and Repetition Control + +#### `--benchmark_min_time=` (BENCHMARK_MIN_TIME) + +Specifies the minimum amount of time (in seconds) that each benchmark should run. For CPU-time based tests, this is the lower bound on the total CPU time used by all threads that make up the test. For real-time based tests, this is the lower bound on the elapsed time of the benchmark execution, regardless of number of threads. + +**Default:** `0.5` seconds + +**Example:** +```bash +$ ./benchmark --benchmark_min_time=1.0 +``` + +#### `--benchmark_min_warmup_time=` (BENCHMARK_MIN_WARMUP_TIME) + +Minimum number of seconds a benchmark should be run before results should be taken into account. This can be necessary for benchmarks of code which needs to fill some form of cache before performance is of interest. Results gathered within this period are discarded and not used for the reported result. + +**Default:** `0.0` seconds + +**Example:** +```bash +$ ./benchmark --benchmark_min_warmup_time=0.5 +``` + +#### `--benchmark_repetitions=` (BENCHMARK_REPETITIONS) + +The number of runs of each benchmark. If greater than 1, the mean and standard deviation of the runs will be reported. + +**Default:** `1` + +**Example:** +```bash +$ ./benchmark --benchmark_repetitions=5 +``` + +### Output Formatting + +#### `--benchmark_format=` (BENCHMARK_FORMAT) + +The format to use for console output. Valid values are 'console', 'json', or 'csv'. See [Output Formats](#output-formats) for more details. + +**Default:** `console` + +**Example:** +```bash +$ ./benchmark --benchmark_format=json +``` + +#### `--benchmark_out=` (BENCHMARK_OUT) + +The file to write additional output to. The output format is controlled by `--benchmark_out_format`. Specifying this option does not suppress console output. + +**Example:** +```bash +$ ./benchmark --benchmark_out=results.json +``` + +#### `--benchmark_out_format=` (BENCHMARK_OUT_FORMAT) + +The format to use for file output specified by `--benchmark_out`. Valid values are 'console', 'json', or 'csv'. + +**Default:** `json` + +**Example:** +```bash +$ ./benchmark --benchmark_out=results.csv --benchmark_out_format=csv +``` + +#### `--benchmark_color=` (BENCHMARK_COLOR) + +Whether to use colors in the output. Valid values are 'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use colors if the output is being sent to a terminal and the TERM environment variable is set to a terminal type that supports colors. + +**Default:** `auto` + +**Example:** +```bash +$ ./benchmark --benchmark_color=false +``` + +#### `--benchmark_time_unit=` (BENCHMARK_TIME_UNIT) + +Set the default time unit to use for reports. Valid values are 'ns' (nanoseconds), 'us' (microseconds), 'ms' (milliseconds), or 's' (seconds). + +**Default:** (empty, uses automatic selection) + +**Example:** +```bash +$ ./benchmark --benchmark_time_unit=us +``` + +### Reporting Options + +#### `--benchmark_report_aggregates_only` (BENCHMARK_REPORT_AGGREGATES_ONLY) + +When enabled, only the mean, standard deviation, and other statistics are reported for repeated benchmarks. This affects all reporters (both console and file output). + +**Default:** `false` + +**Example:** +```bash +$ ./benchmark --benchmark_repetitions=5 --benchmark_report_aggregates_only +``` + +#### `--benchmark_display_aggregates_only` (BENCHMARK_DISPLAY_AGGREGATES_ONLY) + +When enabled, only the mean, standard deviation, and other statistics are displayed for repeated benchmarks. Unlike `--benchmark_report_aggregates_only`, this only affects the display (console) reporter, not the file reporter, which will still contain all output. + +**Default:** `false` + +**Example:** +```bash +$ ./benchmark --benchmark_repetitions=5 --benchmark_display_aggregates_only +``` + +#### `--benchmark_counters_tabular` (BENCHMARK_COUNTERS_TABULAR) + +Whether to use tabular format when printing user counters to the console. Valid values: 'true'/'yes'/1, 'false'/'no'/0. + +**Default:** `false` + +**Example:** +```bash +$ ./benchmark --benchmark_counters_tabular=true +``` + +### Performance Counters and Context + +#### `--benchmark_perf_counters=` (BENCHMARK_PERF_COUNTERS) + +List of additional performance counters to collect, in libpfm format. For more information about libpfm, see the [libpfm documentation](https://man7.org/linux/man-pages/man3/libpfm.3.html). + +**Example:** +```bash +$ ./benchmark --benchmark_perf_counters=cycles,instructions,cache-misses +``` + +#### `--benchmark_context=` (BENCHMARK_CONTEXT) + +Extra context to include in the output, formatted as comma-separated key-value pairs. This context is included in the JSON output's `context` object. + +**Example:** +```bash +$ ./benchmark --benchmark_context=compiler=clang,version=13 +``` + +### Miscellaneous + +#### `-v` (V) + +The level of verbose logging to output. Higher values produce more verbose output. + +**Default:** `0` + +**Example:** +```bash +$ ./benchmark -v +``` @@ -77,48 +287,89 @@ tabular data on stdout. Example tabular output looks like: ``` Benchmark Time(ns) CPU(ns) Iterations ---------------------------------------------------------------------- -BM_SetInsert/1024/1 28928 29349 23853 133.097kB/s 33.2742k items/s -BM_SetInsert/1024/8 32065 32913 21375 949.487kB/s 237.372k items/s -BM_SetInsert/1024/10 33157 33648 21431 1.13369MB/s 290.225k items/s +BM_SetInsert/1024/1 28928 29349 23853 133.097kiB/s 33.2742k items/s +BM_SetInsert/1024/8 32065 32913 21375 949.487kiB/s 237.372k items/s +BM_SetInsert/1024/10 33157 33648 21431 1.13369MiB/s 290.225k items/s ``` -The JSON format outputs human readable json split into two top level attributes. -The `context` attribute contains information about the run in general, including -information about the CPU and the date. -The `benchmarks` attribute contains a list of every benchmark run. Example json -output looks like: +The JSON format outputs human readable JSON split into two top level +attributes: `context` and `benchmarks`. This format is useful for tools that +need to consume benchmark results without parsing console output. + +The `context` object contains information about the run in general, including +the date, host, CPU, caches, load average, benchmark library version, and +`json_schema_version`. Extra context added with `benchmark::AddCustomContext` or +`--benchmark_context` is emitted as additional string fields in `context`. + +The `benchmarks` array contains an object for each benchmark result. Iteration +results commonly include fields such as `name`, `run_name`, `run_type`, +`iterations`, `real_time`, `cpu_time`, `time_unit`, and `threads`. Depending on +benchmark configuration, result objects can also include aggregate fields, +asymptotic complexity fields, skip/error fields, memory metrics, labels, user +counters, and user-requested performance counters. + +User counters, including rates such as `bytes_per_second` and +`items_per_second`, are emitted as additional numeric fields on the benchmark +object. User-requested performance counters are reported the same way. + +The JSON output may gain new fields over time. Consumers should ignore unknown +fields and tolerate optional fields being absent. This allows the format to be +extended while preserving compatibility for existing consumers. + +An abbreviated example JSON output looks like: ```json { "context": { "date": "2015/03/17-18:40:25", + "host_name": "my-host", "num_cpus": 40, "mhz_per_cpu": 2801, "cpu_scaling_enabled": false, - "build_type": "debug" + "caches": [ + { + "type": "Data", + "level": 1, + "size": 32768, + "num_sharing": 2 + } + ], + "load_avg": [], + "library_version": "vX.Y.Z", + "library_build_type": "debug", + "json_schema_version": 1 }, "benchmarks": [ { "name": "BM_SetInsert/1024/1", + "run_name": "BM_SetInsert/1024/1", + "run_type": "iteration", "iterations": 94877, "real_time": 29275, "cpu_time": 29836, + "time_unit": "ns", "bytes_per_second": 134066, "items_per_second": 33516 }, { "name": "BM_SetInsert/1024/8", + "run_name": "BM_SetInsert/1024/8", + "run_type": "iteration", "iterations": 21609, "real_time": 32317, "cpu_time": 32429, + "time_unit": "ns", "bytes_per_second": 986770, "items_per_second": 246693 }, { "name": "BM_SetInsert/1024/10", + "run_name": "BM_SetInsert/1024/10", + "run_type": "iteration", "iterations": 21393, "real_time": 32724, "cpu_time": 33355, + "time_unit": "ns", "bytes_per_second": 1199226, "items_per_second": 299807 } @@ -162,6 +413,13 @@ line interface or by setting environment variables before execution. For every prevails). A complete list of CLI options is available running benchmarks with the `--help` switch. +### Dry runs + +To confirm that benchmarks can run successfully without needing to wait for +multiple repetitions and iterations, the `--benchmark_dry_run` flag can be +used. This will run the benchmarks as normal, but for 1 iteration and 1 +repetition only. + ## Running a Subset of Benchmarks @@ -240,6 +498,32 @@ iterations is at least one, not more than 1e9, until CPU time is greater than the minimum time, or the wallclock time is 5x minimum time. The minimum time is set per benchmark by calling `MinTime` on the registered benchmark object. +The minimum time can also be set for all benchmarks with the +`--benchmark_min_time=` command-line option. This flag supports two +forms: + +* `--benchmark_min_time=s` sets the minimum running time for each + benchmark repetition in seconds. +* `--benchmark_min_time=x` runs each benchmark repetition for an + explicit number of iterations instead of using the dynamic time-based + iteration selection. This applies to benchmarks that do not already specify + an explicit iteration count in code. + +For compatibility, bare numeric values such as `--benchmark_min_time=0.5` are +also interpreted as seconds, but the explicit `s` suffix is preferred for +clarity. + +For example: + +```bash +$ ./run_benchmarks.x --benchmark_min_time=0.5s +$ ./run_benchmarks.x --benchmark_min_time=100x +``` + +If a benchmark specifies its own `MinTime()` or `Iterations()` in code, those +per-benchmark settings take precedence over the corresponding +`--benchmark_min_time` command-line forms. + Furthermore warming up a benchmark might be necessary in order to get stable results because of e.g caching effects of the code under benchmark. Warming up means running the benchmark a given amount of time, before @@ -268,10 +552,12 @@ information about the machine on which the benchmarks are run. Global setup/teardown specific to each benchmark can be done by passing a callback to Setup/Teardown: -The setup/teardown callbacks will be invoked once for each benchmark. -If the benchmark is multi-threaded (will run in k threads), they will be invoked exactly once before -each run with k threads. -If the benchmark uses different size groups of threads, the above will be true for each size group. +The setup/teardown callbacks will be invoked once for each benchmark. If the +benchmark is multi-threaded (will run in k threads), they will be invoked +exactly once before each run with k threads. + +If the benchmark uses different size groups of threads, the above will be true +for each size group. Eg., @@ -314,7 +600,7 @@ static void BM_memcpy(benchmark::State& state) { delete[] src; delete[] dst; } -BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10); +BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(4<<10)->Arg(8<<10); ``` The preceding code is quite repetitive, and can be replaced with the following @@ -343,7 +629,8 @@ the performance of `std::vector` initialization for uniformly increasing sizes. static void BM_DenseRange(benchmark::State& state) { for(auto _ : state) { std::vector v(state.range(0), state.range(0)); - benchmark::DoNotOptimize(v.data()); + auto data = v.data(); + benchmark::DoNotOptimize(data); benchmark::ClobberMemory(); } } @@ -383,17 +670,17 @@ short-hand. The following macro will pick a few appropriate arguments in the product of the two specified ranges and will generate a benchmark for each such pair. -{% raw %} + ```c++ BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}}); ``` -{% endraw %} + Some benchmarks may require specific argument values that cannot be expressed with `Ranges`. In this case, `ArgsProduct` offers the ability to generate a benchmark input for each combination in the product of the supplied vectors. -{% raw %} + ```c++ BENCHMARK(BM_SetInsert) ->ArgsProduct({{1<<10, 3<<10, 8<<10}, {20, 40, 60, 80}}) @@ -412,7 +699,7 @@ BENCHMARK(BM_SetInsert) ->Args({3<<10, 80}) ->Args({8<<10, 80}); ``` -{% endraw %} + For the most common scenarios, helper methods for creating a list of integers for a given sparse or dense range are provided. @@ -437,7 +724,7 @@ benchmark. The following example enumerates a dense range on one parameter, and a sparse range on the second. ```c++ -static void CustomArguments(benchmark::internal::Benchmark* b) { +static void CustomArguments(benchmark::Benchmark* b) { for (int i = 0; i <= 10; ++i) for (int j = 32; j <= 1024*1024; j *= 8) b->Args({i, j}); @@ -445,9 +732,39 @@ static void CustomArguments(benchmark::internal::Benchmark* b) { BENCHMARK(BM_SetInsert)->Apply(CustomArguments); ``` +### Naming Benchmark Arguments + +When a benchmark takes one or more numeric arguments, the generated benchmark +names can be made easier to read by naming those arguments. Use `ArgName` for a +single argument and `ArgNames` for multiple arguments. + +```c++ +BENCHMARK(BM_memcpy)->Range(8, 512)->ArgName("bytes"); +``` + +This changes names such as `BM_memcpy/8` and `BM_memcpy/512` to +`BM_memcpy/bytes:8` and `BM_memcpy/bytes:512`. + +For benchmarks with more than one argument, each name labels the corresponding +argument position. + + +```c++ +BENCHMARK(BM_SetInsert) + ->Args({100, 128}) + ->Args({200, 512}) + ->ArgNames({"size", "inserts"}); +``` + + +This produces names such as `BM_SetInsert/size:100/inserts:128` and +`BM_SetInsert/size:200/inserts:512`. Empty argument names are allowed and leave +that argument value unlabeled, for example `ArgNames({"size", ""})` produces +names like `BM_SetInsert/size:100/128`. + ### Passing Arbitrary Arguments to a Benchmark -In C++11 it is possible to define a benchmark that takes an arbitrary number +It is possible to define a benchmark that takes an arbitrary number of extra arguments. The `BENCHMARK_CAPTURE(func, test_case_name, ...args)` macro creates a benchmark that invokes `func` with the `benchmark::State` as the first argument followed by the specified `args...`. @@ -476,6 +793,25 @@ BENCHMARK_CAPTURE(BM_takes_args, int_test, 42, 43); Note that elements of `...args` may refer to global variables. Users should avoid modifying global state inside of a benchmark. +### Naming a Benchmark Without Capturing Arguments + +If you only need to give a benchmark a custom name (without passing extra +arguments), use `BENCHMARK_NAMED(func, test_case_name)`. Unlike +`BENCHMARK_CAPTURE`, this macro does not create a lambda, which avoids +compiler and linker scalability issues when registering thousands of +benchmarks. + +```c++ +void BM_Foo(benchmark::State& state) { + for (auto _ : state) {} +} +// Registers a benchmark named "BM_Foo/my_variant" +BENCHMARK_NAMED(BM_Foo, my_variant); +``` + +Use `BENCHMARK_CAPTURE` when you need to pass extra arguments; use +`BENCHMARK_NAMED` when you only need the name. + ## Calculating Asymptotic Complexity (Big O) @@ -489,7 +825,8 @@ static void BM_StringCompare(benchmark::State& state) { std::string s1(state.range(0), '-'); std::string s2(state.range(0), '-'); for (auto _ : state) { - benchmark::DoNotOptimize(s1.compare(s2)); + auto comparison_result = s1.compare(s2); + benchmark::DoNotOptimize(comparison_result); } state.SetComplexityN(state.range(0)); } @@ -547,26 +884,47 @@ template void BM_Sequential(benchmark::State& state) { state.SetBytesProcessed( static_cast(state.iterations())*state.range(0)); } -// C++03 -BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue)->Range(1<<0, 1<<10); -// C++11 or newer, you can use the BENCHMARK macro with template parameters: +// You can use the BENCHMARK macro with template parameters: BENCHMARK(BM_Sequential>)->Range(1<<0, 1<<10); +// Old, legacy verbose C++03 syntax: +BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue)->Range(1<<0, 1<<10); + ``` Three macros are provided for adding benchmark templates. ```c++ -#ifdef BENCHMARK_HAS_CXX11 #define BENCHMARK(func<...>) // Takes any number of parameters. -#else // C++ < C++11 -#define BENCHMARK_TEMPLATE(func, arg1) -#endif #define BENCHMARK_TEMPLATE1(func, arg1) #define BENCHMARK_TEMPLATE2(func, arg1, arg2) ``` + + +## Templated Benchmarks that take arguments + +Sometimes there is a need to template benchmarks, and provide arguments to them. + +```c++ +template void BM_Sequential_With_Step(benchmark::State& state, int step) { + Q q; + typename Q::value_type v; + for (auto _ : state) { + for (int i = state.range(0); i-=step; ) + q.push(v); + for (int e = state.range(0); e-=step; ) + q.Wait(&v); + } + // actually messages, not bytes: + state.SetBytesProcessed( + static_cast(state.iterations())*state.range(0)); +} + +BENCHMARK_TEMPLATE1_CAPTURE(BM_Sequential, WaitQueue, Step1, 1)->Range(1<<0, 1<<10); +``` + ## Fixtures @@ -584,27 +942,29 @@ For Example: ```c++ class MyFixture : public benchmark::Fixture { public: - void SetUp(const ::benchmark::State& state) { + void SetUp(::benchmark::State& state) { } - void TearDown(const ::benchmark::State& state) { + void TearDown(::benchmark::State& state) { } }; +// Defines and registers `FooTest` using the class `MyFixture`. BENCHMARK_F(MyFixture, FooTest)(benchmark::State& st) { for (auto _ : st) { ... } } +// Only defines `BarTest` using the class `MyFixture`. BENCHMARK_DEFINE_F(MyFixture, BarTest)(benchmark::State& st) { for (auto _ : st) { ... } } -/* BarTest is NOT registered */ +// `BarTest` is NOT registered. BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2); -/* BarTest is now registered */ +// `BarTest` is now registered. ``` ### Templated Fixtures @@ -620,19 +980,70 @@ For example: template class MyFixture : public benchmark::Fixture {}; +// Defines and registers `IntTest` using the class template `MyFixture`. BENCHMARK_TEMPLATE_F(MyFixture, IntTest, int)(benchmark::State& st) { for (auto _ : st) { ... } } +// Only defines `DoubleTest` using the class template `MyFixture`. BENCHMARK_TEMPLATE_DEFINE_F(MyFixture, DoubleTest, double)(benchmark::State& st) { for (auto _ : st) { ... } } - +// `DoubleTest` is NOT registered. BENCHMARK_REGISTER_F(MyFixture, DoubleTest)->Threads(2); +// `DoubleTest` is now registered. +``` + +If you want to use a method template for your fixtures, +which you instantiate afterward, use the following macros: + +* `BENCHMARK_TEMPLATE_METHOD_F(ClassName, Method)` +* `BENCHMARK_TEMPLATE_INSTANTIATE_F(ClassName, Method, ...)` + +With these macros you can define one method for several instantiations. +Example (using `MyFixture` from above): + +```c++ +// Defines `Test` using the class template `MyFixture`. +BENCHMARK_TEMPLATE_METHOD_F(MyFixture, Test)(benchmark::State& st) { + for (auto _ : st) { + ... + } +} + +// Instantiates and registers the benchmark `MyFixture::Test`. +BENCHMARK_TEMPLATE_INSTANTIATE_F(MyFixture, Test, int)->Threads(2); +// Instantiates and registers the benchmark `MyFixture::Test`. +BENCHMARK_TEMPLATE_INSTANTIATE_F(MyFixture, Test, double)->Threads(4); +``` + +Inside the method definition of `BENCHMARK_TEMPLATE_METHOD_F` the type `Base` refers +to the type of the instantiated fixture. +Accesses to members of the fixture must be prefixed by `this->`. + +`BENCHMARK_TEMPLATE_METHOD_F`and `BENCHMARK_TEMPLATE_INSTANTIATE_F` can only be used, +if the fixture does not use non-type template parameters. +If you want to pass values as template parameters, use e.g. `std::integral_constant`. +For example: + +```c++ +template +class SizedFixture : public benchmark::Fixture { + static constexpr auto Size = Sz::value; + int myValue; +}; + +BENCHMARK_TEMPLATE_METHOD_F(SizedFixture, Test)(benchmark::State& st) { + for (auto _ : st) { + this->myValue = Base::Size; + } +} + +BENCHMARK_TEMPLATE_INSTANTIATE_F(SizedFixture, Test, std::integral_constant<5>)->Threads(2); ``` @@ -659,10 +1070,6 @@ and `Counter` values. The latter is a `double`-like class, via an implicit conversion to `double&`. Thus you can use all of the standard arithmetic assignment operators (`=,+=,-=,*=,/=`) to change the value of each counter. -In multithreaded benchmarks, each counter is set on the calling thread only. -When the benchmark finishes, the counters from each thread will be summed; -the resulting sum is the value which will be shown for the benchmark. - The `Counter` constructor accepts three parameters: the value as a `double` ; a bit flag which allows you to show counters as rates, and/or as per-thread iteration, and/or as per-thread averages, and/or iteration invariants, @@ -695,19 +1102,21 @@ is 1k a 1000 (default, `benchmark::Counter::OneK::kIs1000`), or 1024 state.counters["BytesProcessed"] = Counter(state.range(0), benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1024); ``` -When you're compiling in C++11 mode or later you can use `insert()` with -`std::initializer_list`: +You can use `insert()` with `std::initializer_list`: -{% raw %} + ```c++ - // With C++11, this can be done: state.counters.insert({{"Foo", numFoos}, {"Bar", numBars}, {"Baz", numBazs}}); // ... instead of: state.counters["Foo"] = numFoos; state.counters["Bar"] = numBars; state.counters["Baz"] = numBazs; ``` -{% endraw %} + + +In multithreaded benchmarks, each counter is set on the calling thread only. +When the benchmark finishes, the counters from each thread will be summed. +Counters that are configured with `kIsRate`, will report the average rate across all threads, while `kAvgThreadsRate` counters will report the average rate per thread. ### Counter Reporting @@ -823,6 +1232,46 @@ BENCHMARK(BM_test)->Range(8, 8<<10)->UseRealTime(); Without `UseRealTime`, CPU time is used by default. +### Manual Multithreaded Benchmarks + +Google/benchmark uses `std::thread` as multithreading environment per default. +If you want to use another multithreading environment (e.g. OpenMP), you can provide +a factory function to your benchmark using the `ThreadRunner` function. +The factory function takes the number of threads as argument and creates a custom class +derived from `benchmark::ThreadRunnerBase`. +This custom class must override the function +`void RunThreads(const std::function& fn)`. +`RunThreads` is called by the main thread and spawns the requested number of threads. +Each spawned thread must call `fn(thread_index)`, where `thread_index` is its own +thread index. Before `RunThreads` returns, all spawned threads must be joined. +```c++ +class OpenMPThreadRunner : public benchmark::ThreadRunnerBase +{ + OpenMPThreadRunner(int num_threads) + : num_threads_(num_threads) + {} + + void RunThreads(const std::function& fn) final + { +#pragma omp parallel num_threads(num_threads_) + fn(omp_get_thread_num()); + } + +private: + int num_threads_; +}; + +BENCHMARK(BM_MultiThreaded) + ->ThreadRunner([](int num_threads) { + return std::make_unique(num_threads); + }) + ->Threads(1)->Threads(2)->Threads(4); +``` +The above example creates a parallel OpenMP region before it enters `BM_MultiThreaded`. +The actual benchmark code can remain the same and is therefore not tied to a specific +thread runner. The measurement does not include the time for creating and joining the +threads. + ## CPU Timers @@ -854,7 +1303,7 @@ BENCHMARK(BM_OpenMP)->Range(8, 8<<10); // Measure the user-visible time, the wall clock (literally, the time that // has passed on the clock on the wall), use it to decide for how long to -// run the benchmark loop. This will always be meaningful, an will match the +// run the benchmark loop. This will always be meaningful, and will match the // time spent by the main thread in single-threaded case, in general decreasing // with the number of internal threads doing the work. BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->UseRealTime(); @@ -876,7 +1325,7 @@ is measured. But sometimes, it is necessary to do some work inside of that loop, every iteration, but without counting that time to the benchmark time. That is possible, although it is not recommended, since it has high overhead. -{% raw %} + ```c++ static void BM_SetInsert_With_Timer_Control(benchmark::State& state) { std::set data; @@ -891,7 +1340,30 @@ static void BM_SetInsert_With_Timer_Control(benchmark::State& state) { } BENCHMARK(BM_SetInsert_With_Timer_Control)->Ranges({{1<<10, 8<<10}, {128, 512}}); ``` -{% endraw %} + + +For convenience, a `ScopedPauseTiming` class is provided to manage pausing and +resuming timers within a scope. This is less error-prone than manually calling +`PauseTiming` and `ResumeTiming`. + + +```c++ +static void BM_SetInsert_With_Scoped_Timer_Control(benchmark::State& state) { + std::set data; + for (auto _ : state) { + { + benchmark::ScopedPauseTiming pause(state); // Pauses timing + data = ConstructRandomSet(state.range(0)); + } // Timing resumes automatically when 'pause' goes out of scope + + // The rest will be measured. + for (int j = 0; j < state.range(1); ++j) + data.insert(RandomNumber()); + } +} +BENCHMARK(BM_SetInsert_With_Scoped_Timer_Control)->Ranges({{1<<10, 8<<10}, {128, 512}}); +``` + @@ -979,17 +1451,33 @@ in any way. `` may even be removed entirely when the result is already known. For example: ```c++ - /* Example 1: `` is removed entirely. */ + // Example 1: `` is removed entirely. int foo(int x) { return x + 42; } while (...) DoNotOptimize(foo(0)); // Optimized to DoNotOptimize(42); - /* Example 2: Result of '' is only reused */ + // Example 2: Result of '' is only reused. int bar(int) __attribute__((const)); while (...) DoNotOptimize(bar(0)); // Optimized to: // int __result__ = bar(0); // while (...) DoNotOptimize(__result__); ``` +Since that is not the behaviour the user intended, +such problematic cases will result in a diagnostic +at compile time. The correct approach, for an expression result, +is to use an intermediate local variable: + +```c++ + // Avoid: may call the deprecated const-reference overload. + while (...) DoNotOptimize(foo(0)); + + // Prefer: materialize the result, then pass the local lvalue. + while (...) { + auto result = foo(0); + DoNotOptimize(result); + } +``` + The second tool for preventing optimizations is `ClobberMemory()`. In essence `ClobberMemory()` forces the compiler to perform all pending writes to global memory. Memory managed by block scope objects must be "escaped" using @@ -1002,7 +1490,8 @@ static void BM_vector_push_back(benchmark::State& state) { for (auto _ : state) { std::vector v; v.reserve(1); - benchmark::DoNotOptimize(v.data()); // Allow v.data() to be clobbered. + auto data = v.data(); // Allow v.data() to be clobbered. Pass as non-const + benchmark::DoNotOptimize(data); // lvalue to avoid undesired compiler optimizations v.push_back(42); benchmark::ClobberMemory(); // Force 42 to be written to memory. } @@ -1060,6 +1549,7 @@ void BM_spin_empty(benchmark::State& state) { } BENCHMARK(BM_spin_empty) + ->Repetitions(3) // or add option --benchmark_repetitions=3 ->ComputeStatistics("max", [](const std::vector& v) -> double { return *(std::max_element(std::begin(v), std::end(v))); }) @@ -1079,8 +1569,9 @@ void BM_spin_empty(benchmark::State& state) { } BENCHMARK(BM_spin_empty) + ->Repetitions(3) // or add option --benchmark_repetitions=3 ->ComputeStatistics("ratio", [](const std::vector& v) -> double { - return std::begin(v) / std::end(v); + return v.front() / v.back(); }, benchmark::StatisticUnit::kPercentage) ->Arg(512); ``` @@ -1100,6 +1591,21 @@ a report on the number of allocations, bytes used, etc. This data will then be reported alongside other performance data, currently only when using JSON output. + + +## Profiling + +It's often useful to also profile benchmarks in particular ways, in addition to +CPU performance. For this reason, benchmark offers the `RegisterProfilerManager` +method that allows a custom `ProfilerManager` to be injected. + +If set, the `ProfilerManager::AfterSetupStart` and +`ProfilerManager::BeforeTeardownStop` methods will be called at the start and +end of a separate benchmark run to allow user code to collect and report +user-provided profile metrics. + +Output collected from this profiling run must be reported separately. + ## Using RegisterBenchmark(name, fn, args...) @@ -1122,6 +1628,7 @@ For Example: auto BM_test = [](benchmark::State& st, auto Inputs) { /* ... */ }; int main(int argc, char** argv) { + benchmark::MaybeReenterWithoutASLR(argc, argv); for (auto& test_input : { /* ... */ }) benchmark::RegisterBenchmark(test_input.name(), BM_test, test_input); benchmark::Initialize(&argc, argv); @@ -1136,7 +1643,7 @@ int main(int argc, char** argv) { When errors caused by external influences, such as file I/O and network communication, occur within a benchmark the -`State::SkipWithError(const char* msg)` function can be used to skip that run +`State::SkipWithError(const std::string& msg)` function can be used to skip that run of benchmark and report the error. Note that only future iterations of the `KeepRunning()` are skipped. For the ranged-for version of the benchmark loop Users must explicitly exit the loop, otherwise all iterations will be performed. @@ -1186,7 +1693,7 @@ static void BM_test_ranged_fo(benchmark::State & state) { ## A Faster KeepRunning Loop -In C++11 mode, a ranged-based for loop should be used in preference to +A ranged-based for loop should be used in preference to the `KeepRunning` loop for running the benchmarks. For example: ```c++ @@ -1247,39 +1754,12 @@ the benchmark loop should be preferred. If you see this error: ``` -***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead. +***WARNING*** CPU scaling is enabled, the benchmark real time measurements may +be noisy and will incur extra overhead. ``` you might want to disable the CPU frequency scaling while running the -benchmark. Exactly how to do this depends on the Linux distribution, -desktop environment, and installed programs. Specific details are a moving -target, so we will not attempt to exhaustively document them here. - -One simple option is to use the `cpupower` program to change the -performance governor to "performance". This tool is maintained along with -the Linux kernel and provided by your distribution. - -It must be run as root, like this: - -```bash -sudo cpupower frequency-set --governor performance -``` - -After this you can verify that all CPUs are using the performance governor -by running this command: - -```bash -cpupower frequency-info -o proc -``` - -The benchmarks you subsequently run will have less variance. - -Note that changing the governor in this way will not persist across -reboots. To set the governor back, run the first command again with the -governor your system usually runs with, which varies. +benchmark, as well as consider other ways to stabilize the performance of +your system while benchmarking. -If you find yourself doing this often, there are probably better options -than running the commands above. Some approaches allow you to do this -without root access, or by using a GUI, etc. The Arch Wiki [Cpu frequency -scaling](https://wiki.archlinux.org/title/CPU_frequency_scaling) page is a -good place to start looking for options. +See [Reducing Variance](reducing_variance.md) for more information. diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index a4fc52df6f..e065db35cf 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -12,1863 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Support for registering benchmarks for functions. - -/* Example usage: -// Define a function that executes the code to be measured a -// specified number of times: -static void BM_StringCreation(benchmark::State& state) { - for (auto _ : state) - std::string empty_string; -} - -// Register the function as a benchmark -BENCHMARK(BM_StringCreation); - -// Define another benchmark -static void BM_StringCopy(benchmark::State& state) { - std::string x = "hello"; - for (auto _ : state) - std::string copy(x); -} -BENCHMARK(BM_StringCopy); - -// Augment the main() program to invoke benchmarks if specified -// via the --benchmark_filter command line flag. E.g., -// my_unittest --benchmark_filter=all -// my_unittest --benchmark_filter=BM_StringCreation -// my_unittest --benchmark_filter=String -// my_unittest --benchmark_filter='Copy|Creation' -int main(int argc, char** argv) { - benchmark::Initialize(&argc, argv); - benchmark::RunSpecifiedBenchmarks(); - benchmark::Shutdown(); - return 0; -} - -// Sometimes a family of microbenchmarks can be implemented with -// just one routine that takes an extra argument to specify which -// one of the family of benchmarks to run. For example, the following -// code defines a family of microbenchmarks for measuring the speed -// of memcpy() calls of different lengths: - -static void BM_memcpy(benchmark::State& state) { - char* src = new char[state.range(0)]; char* dst = new char[state.range(0)]; - memset(src, 'x', state.range(0)); - for (auto _ : state) - memcpy(dst, src, state.range(0)); - state.SetBytesProcessed(state.iterations() * state.range(0)); - delete[] src; delete[] dst; -} -BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10); - -// The preceding code is quite repetitive, and can be replaced with the -// following short-hand. The following invocation will pick a few -// appropriate arguments in the specified range and will generate a -// microbenchmark for each such argument. -BENCHMARK(BM_memcpy)->Range(8, 8<<10); - -// You might have a microbenchmark that depends on two inputs. For -// example, the following code defines a family of microbenchmarks for -// measuring the speed of set insertion. -static void BM_SetInsert(benchmark::State& state) { - set data; - for (auto _ : state) { - state.PauseTiming(); - data = ConstructRandomSet(state.range(0)); - state.ResumeTiming(); - for (int j = 0; j < state.range(1); ++j) - data.insert(RandomNumber()); - } -} -BENCHMARK(BM_SetInsert) - ->Args({1<<10, 128}) - ->Args({2<<10, 128}) - ->Args({4<<10, 128}) - ->Args({8<<10, 128}) - ->Args({1<<10, 512}) - ->Args({2<<10, 512}) - ->Args({4<<10, 512}) - ->Args({8<<10, 512}); - -// The preceding code is quite repetitive, and can be replaced with -// the following short-hand. The following macro will pick a few -// appropriate arguments in the product of the two specified ranges -// and will generate a microbenchmark for each such pair. -BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}}); - -// For more complex patterns of inputs, passing a custom function -// to Apply allows programmatic specification of an -// arbitrary set of arguments to run the microbenchmark on. -// The following example enumerates a dense range on -// one parameter, and a sparse range on the second. -static void CustomArguments(benchmark::internal::Benchmark* b) { - for (int i = 0; i <= 10; ++i) - for (int j = 32; j <= 1024*1024; j *= 8) - b->Args({i, j}); -} -BENCHMARK(BM_SetInsert)->Apply(CustomArguments); - -// Templated microbenchmarks work the same way: -// Produce then consume 'size' messages 'iters' times -// Measures throughput in the absence of multiprogramming. -template int BM_Sequential(benchmark::State& state) { - Q q; - typename Q::value_type v; - for (auto _ : state) { - for (int i = state.range(0); i--; ) - q.push(v); - for (int e = state.range(0); e--; ) - q.Wait(&v); - } - // actually messages, not bytes: - state.SetBytesProcessed(state.iterations() * state.range(0)); -} -BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue)->Range(1<<0, 1<<10); - -Use `Benchmark::MinTime(double t)` to set the minimum time used to run the -benchmark. This option overrides the `benchmark_min_time` flag. - -void BM_test(benchmark::State& state) { - ... body ... -} -BENCHMARK(BM_test)->MinTime(2.0); // Run for at least 2 seconds. - -In a multithreaded test, it is guaranteed that none of the threads will start -until all have reached the loop start, and all will have finished before any -thread exits the loop body. As such, any global setup or teardown you want to -do can be wrapped in a check against the thread index: - -static void BM_MultiThreaded(benchmark::State& state) { - if (state.thread_index() == 0) { - // Setup code here. - } - for (auto _ : state) { - // Run the test as normal. - } - if (state.thread_index() == 0) { - // Teardown code here. - } -} -BENCHMARK(BM_MultiThreaded)->Threads(4); - - -If a benchmark runs a few milliseconds it may be hard to visually compare the -measured times, since the output data is given in nanoseconds per default. In -order to manually set the time unit, you can specify it manually: - -BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); -*/ - #ifndef BENCHMARK_BENCHMARK_H_ #define BENCHMARK_BENCHMARK_H_ -// The _MSVC_LANG check should detect Visual Studio 2015 Update 3 and newer. -#if __cplusplus >= 201103L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201103L) -#define BENCHMARK_HAS_CXX11 -#endif - -// This _MSC_VER check should detect VS 2017 v15.3 and newer. -#if __cplusplus >= 201703L || \ - (defined(_MSC_VER) && _MSC_VER >= 1911 && _MSVC_LANG >= 201703L) -#define BENCHMARK_HAS_CXX17 -#endif - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "benchmark/export.h" - -#if defined(BENCHMARK_HAS_CXX11) -#include -#include -#include -#include -#endif - -#if defined(_MSC_VER) -#include // for _ReadWriteBarrier -#endif - -#ifndef BENCHMARK_HAS_CXX11 -#define BENCHMARK_DISALLOW_COPY_AND_ASSIGN(TypeName) \ - TypeName(const TypeName&); \ - TypeName& operator=(const TypeName&) -#else -#define BENCHMARK_DISALLOW_COPY_AND_ASSIGN(TypeName) \ - TypeName(const TypeName&) = delete; \ - TypeName& operator=(const TypeName&) = delete -#endif - -#ifdef BENCHMARK_HAS_CXX17 -#define BENCHMARK_UNUSED [[maybe_unused]] -#elif defined(__GNUC__) || defined(__clang__) -#define BENCHMARK_UNUSED __attribute__((unused)) -#else -#define BENCHMARK_UNUSED -#endif - -#if defined(__GNUC__) || defined(__clang__) -#define BENCHMARK_ALWAYS_INLINE __attribute__((always_inline)) -#define BENCHMARK_NOEXCEPT noexcept -#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) -#elif defined(_MSC_VER) && !defined(__clang__) -#define BENCHMARK_ALWAYS_INLINE __forceinline -#if _MSC_VER >= 1900 -#define BENCHMARK_NOEXCEPT noexcept -#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) -#else -#define BENCHMARK_NOEXCEPT -#define BENCHMARK_NOEXCEPT_OP(x) -#endif -#define __func__ __FUNCTION__ -#else -#define BENCHMARK_ALWAYS_INLINE -#define BENCHMARK_NOEXCEPT -#define BENCHMARK_NOEXCEPT_OP(x) -#endif - -#define BENCHMARK_INTERNAL_TOSTRING2(x) #x -#define BENCHMARK_INTERNAL_TOSTRING(x) BENCHMARK_INTERNAL_TOSTRING2(x) - -// clang-format off -#if defined(__GNUC__) || defined(__clang__) -#define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y) -#define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) -#define BENCHMARK_DISABLE_DEPRECATED_WARNING \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") -#define BENCHMARK_RESTORE_DEPRECATED_WARNING _Pragma("GCC diagnostic pop") -#else -#define BENCHMARK_BUILTIN_EXPECT(x, y) x -#define BENCHMARK_DEPRECATED_MSG(msg) -#define BENCHMARK_WARNING_MSG(msg) \ - __pragma(message(__FILE__ "(" BENCHMARK_INTERNAL_TOSTRING( \ - __LINE__) ") : warning note: " msg)) -#define BENCHMARK_DISABLE_DEPRECATED_WARNING -#define BENCHMARK_RESTORE_DEPRECATED_WARNING -#endif -// clang-format on - -#if defined(__GNUC__) && !defined(__clang__) -#define BENCHMARK_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#endif - -#ifndef __has_builtin -#define __has_builtin(x) 0 -#endif - -#if defined(__GNUC__) || __has_builtin(__builtin_unreachable) -#define BENCHMARK_UNREACHABLE() __builtin_unreachable() -#elif defined(_MSC_VER) -#define BENCHMARK_UNREACHABLE() __assume(false) -#else -#define BENCHMARK_UNREACHABLE() ((void)0) -#endif - -#ifdef BENCHMARK_HAS_CXX11 -#define BENCHMARK_OVERRIDE override -#else -#define BENCHMARK_OVERRIDE -#endif - -#if defined(_MSC_VER) -#pragma warning(push) -// C4251: needs to have dll-interface to be used by clients of class -#pragma warning(disable : 4251) -#endif - -namespace benchmark { -class BenchmarkReporter; - -BENCHMARK_EXPORT void Initialize(int* argc, char** argv, - void (*HelperPrinterf)() = NULL); -BENCHMARK_EXPORT void Shutdown(); - -// Report to stdout all arguments in 'argv' as unrecognized except the first. -// Returns true there is at least on unrecognized argument (i.e. 'argc' > 1). -BENCHMARK_EXPORT bool ReportUnrecognizedArguments(int argc, char** argv); - -// Returns the current value of --benchmark_filter. -BENCHMARK_EXPORT std::string GetBenchmarkFilter(); - -// Sets a new value to --benchmark_filter. (This will override this flag's -// current value). -// Should be called after `benchmark::Initialize()`, as -// `benchmark::Initialize()` will override the flag's value. -BENCHMARK_EXPORT void SetBenchmarkFilter(std::string value); - -// Returns the current value of --v (command line value for verbosity). -BENCHMARK_EXPORT int32_t GetBenchmarkVerbosity(); - -// Creates a default display reporter. Used by the library when no display -// reporter is provided, but also made available for external use in case a -// custom reporter should respect the `--benchmark_format` flag as a fallback -BENCHMARK_EXPORT BenchmarkReporter* CreateDefaultDisplayReporter(); - -// Generate a list of benchmarks matching the specified --benchmark_filter flag -// and if --benchmark_list_tests is specified return after printing the name -// of each matching benchmark. Otherwise run each matching benchmark and -// report the results. -// -// spec : Specify the benchmarks to run. If users do not specify this arg, -// then the value of FLAGS_benchmark_filter -// will be used. -// -// The second and third overload use the specified 'display_reporter' and -// 'file_reporter' respectively. 'file_reporter' will write to the file -// specified -// by '--benchmark_output'. If '--benchmark_output' is not given the -// 'file_reporter' is ignored. -// -// RETURNS: The number of matching benchmarks. -BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks(); -BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks(std::string spec); - -BENCHMARK_EXPORT size_t -RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter); -BENCHMARK_EXPORT size_t -RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, std::string spec); - -BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks( - BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter); -BENCHMARK_EXPORT size_t -RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, - BenchmarkReporter* file_reporter, std::string spec); - -// TimeUnit is passed to a benchmark in order to specify the order of magnitude -// for the measured time. -enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond, kSecond }; - -BENCHMARK_EXPORT TimeUnit GetDefaultTimeUnit(); - -// Sets the default time unit the benchmarks use -// Has to be called before the benchmark loop to take effect -BENCHMARK_EXPORT void SetDefaultTimeUnit(TimeUnit unit); - -// If a MemoryManager is registered (via RegisterMemoryManager()), -// it can be used to collect and report allocation metrics for a run of the -// benchmark. -class MemoryManager { - public: - static const int64_t TombstoneValue; - - struct Result { - Result() - : num_allocs(0), - max_bytes_used(0), - total_allocated_bytes(TombstoneValue), - net_heap_growth(TombstoneValue) {} - - // The number of allocations made in total between Start and Stop. - int64_t num_allocs; - - // The peak memory use between Start and Stop. - int64_t max_bytes_used; - - // The total memory allocated, in bytes, between Start and Stop. - // Init'ed to TombstoneValue if metric not available. - int64_t total_allocated_bytes; - - // The net changes in memory, in bytes, between Start and Stop. - // ie., total_allocated_bytes - total_deallocated_bytes. - // Init'ed to TombstoneValue if metric not available. - int64_t net_heap_growth; - }; - - virtual ~MemoryManager() {} - - // Implement this to start recording allocation information. - virtual void Start() = 0; - - // Implement this to stop recording and fill out the given Result structure. - BENCHMARK_DEPRECATED_MSG("Use Stop(Result&) instead") - virtual void Stop(Result* result) = 0; - - // FIXME(vyng): Make this pure virtual once we've migrated current users. - BENCHMARK_DISABLE_DEPRECATED_WARNING - virtual void Stop(Result& result) { Stop(&result); } - BENCHMARK_RESTORE_DEPRECATED_WARNING -}; - -// Register a MemoryManager instance that will be used to collect and report -// allocation measurements for benchmark runs. -BENCHMARK_EXPORT -void RegisterMemoryManager(MemoryManager* memory_manager); - -// Add a key-value pair to output as part of the context stanza in the report. -BENCHMARK_EXPORT -void AddCustomContext(const std::string& key, const std::string& value); - -namespace internal { -class Benchmark; -class BenchmarkImp; -class BenchmarkFamilies; - -BENCHMARK_EXPORT -void UseCharPointer(char const volatile*); - -// Take ownership of the pointer and register the benchmark. Return the -// registered benchmark. -BENCHMARK_EXPORT Benchmark* RegisterBenchmarkInternal(Benchmark*); - -// Ensure that the standard streams are properly initialized in every TU. -BENCHMARK_EXPORT int InitializeStreams(); -BENCHMARK_UNUSED static int stream_init_anchor = InitializeStreams(); - -} // namespace internal - -#if (!defined(__GNUC__) && !defined(__clang__)) || defined(__pnacl__) || \ - defined(__EMSCRIPTEN__) -#define BENCHMARK_HAS_NO_INLINE_ASSEMBLY -#endif - -// Force the compiler to flush pending writes to global memory. Acts as an -// effective read/write barrier -#ifdef BENCHMARK_HAS_CXX11 -inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { - std::atomic_signal_fence(std::memory_order_acq_rel); -} -#endif - -// The DoNotOptimize(...) function can be used to prevent a value or -// expression from being optimized away by the compiler. This function is -// intended to add little to no overhead. -// See: https://youtu.be/nXaxk27zwlk?t=2441 -#ifndef BENCHMARK_HAS_NO_INLINE_ASSEMBLY -#if !defined(__GNUC__) || defined(__llvm__) || defined(__INTEL_COMPILER) -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { - asm volatile("" : : "r,m"(value) : "memory"); -} - -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { -#if defined(__clang__) - asm volatile("" : "+r,m"(value) : : "memory"); -#else - asm volatile("" : "+m,r"(value) : : "memory"); -#endif -} -#elif defined(BENCHMARK_HAS_CXX11) && (__GNUC__ >= 5) -// Workaround for a bug with full argument copy overhead with GCC. -// See: #1340 and https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105519 -template -inline BENCHMARK_ALWAYS_INLINE - typename std::enable_if::value && - (sizeof(Tp) <= sizeof(Tp*))>::type - DoNotOptimize(Tp const& value) { - asm volatile("" : : "r"(value) : "memory"); -} - -template -inline BENCHMARK_ALWAYS_INLINE - typename std::enable_if::value || - (sizeof(Tp) > sizeof(Tp*))>::type - DoNotOptimize(Tp const& value) { - asm volatile("" : : "m"(value) : "memory"); -} - -template -inline BENCHMARK_ALWAYS_INLINE - typename std::enable_if::value && - (sizeof(Tp) <= sizeof(Tp*))>::type - DoNotOptimize(Tp& value) { - asm volatile("" : "+r"(value) : : "memory"); -} - -template -inline BENCHMARK_ALWAYS_INLINE - typename std::enable_if::value || - (sizeof(Tp) > sizeof(Tp*))>::type - DoNotOptimize(Tp& value) { - asm volatile("" : "+m"(value) : : "memory"); -} - -#else -// Fallback for GCC < 5. Can add some overhead because the compiler is forced -// to use memory operations instead of operations with registers. -// TODO: Remove if GCC < 5 will be unsupported. -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { - asm volatile("" : : "m"(value) : "memory"); -} - -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { - asm volatile("" : "+m"(value) : : "memory"); -} -#endif - -#ifndef BENCHMARK_HAS_CXX11 -inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { - asm volatile("" : : : "memory"); -} -#endif -#elif defined(_MSC_VER) -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { - internal::UseCharPointer(&reinterpret_cast(value)); - _ReadWriteBarrier(); -} - -#ifndef BENCHMARK_HAS_CXX11 -inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { _ReadWriteBarrier(); } -#endif -#else -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { - internal::UseCharPointer(&reinterpret_cast(value)); -} -// FIXME Add ClobberMemory() for non-gnu and non-msvc compilers, before C++11. -#endif - -// This class is used for user-defined counters. -class Counter { - public: - enum Flags { - kDefaults = 0, - // Mark the counter as a rate. It will be presented divided - // by the duration of the benchmark. - kIsRate = 1 << 0, - // Mark the counter as a thread-average quantity. It will be - // presented divided by the number of threads. - kAvgThreads = 1 << 1, - // Mark the counter as a thread-average rate. See above. - kAvgThreadsRate = kIsRate | kAvgThreads, - // Mark the counter as a constant value, valid/same for *every* iteration. - // When reporting, it will be *multiplied* by the iteration count. - kIsIterationInvariant = 1 << 2, - // Mark the counter as a constant rate. - // When reporting, it will be *multiplied* by the iteration count - // and then divided by the duration of the benchmark. - kIsIterationInvariantRate = kIsRate | kIsIterationInvariant, - // Mark the counter as a iteration-average quantity. - // It will be presented divided by the number of iterations. - kAvgIterations = 1 << 3, - // Mark the counter as a iteration-average rate. See above. - kAvgIterationsRate = kIsRate | kAvgIterations, - - // In the end, invert the result. This is always done last! - kInvert = 1 << 31 - }; - - enum OneK { - // 1'000 items per 1k - kIs1000 = 1000, - // 1'024 items per 1k - kIs1024 = 1024 - }; - - double value; - Flags flags; - OneK oneK; - - BENCHMARK_ALWAYS_INLINE - Counter(double v = 0., Flags f = kDefaults, OneK k = kIs1000) - : value(v), flags(f), oneK(k) {} - - BENCHMARK_ALWAYS_INLINE operator double const &() const { return value; } - BENCHMARK_ALWAYS_INLINE operator double&() { return value; } -}; - -// A helper for user code to create unforeseen combinations of Flags, without -// having to do this cast manually each time, or providing this operator. -Counter::Flags inline operator|(const Counter::Flags& LHS, - const Counter::Flags& RHS) { - return static_cast(static_cast(LHS) | - static_cast(RHS)); -} - -// This is the container for the user-defined counters. -typedef std::map UserCounters; - -// BigO is passed to a benchmark in order to specify the asymptotic -// computational -// complexity for the benchmark. In case oAuto is selected, complexity will be -// calculated automatically to the best fit. -enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda }; - -typedef int64_t IterationCount; - -enum StatisticUnit { kTime, kPercentage }; - -// BigOFunc is passed to a benchmark in order to specify the asymptotic -// computational complexity for the benchmark. -typedef double(BigOFunc)(IterationCount); - -// StatisticsFunc is passed to a benchmark in order to compute some descriptive -// statistics over all the measurements of some type -typedef double(StatisticsFunc)(const std::vector&); - -namespace internal { -struct Statistics { - std::string name_; - StatisticsFunc* compute_; - StatisticUnit unit_; - - Statistics(const std::string& name, StatisticsFunc* compute, - StatisticUnit unit = kTime) - : name_(name), compute_(compute), unit_(unit) {} -}; - -class BenchmarkInstance; -class ThreadTimer; -class ThreadManager; -class PerfCountersMeasurement; - -enum AggregationReportMode -#if defined(BENCHMARK_HAS_CXX11) - : unsigned -#else -#endif -{ - // The mode has not been manually specified - ARM_Unspecified = 0, - // The mode is user-specified. - // This may or may not be set when the following bit-flags are set. - ARM_Default = 1U << 0U, - // File reporter should only output aggregates. - ARM_FileReportAggregatesOnly = 1U << 1U, - // Display reporter should only output aggregates - ARM_DisplayReportAggregatesOnly = 1U << 2U, - // Both reporters should only display aggregates. - ARM_ReportAggregatesOnly = - ARM_FileReportAggregatesOnly | ARM_DisplayReportAggregatesOnly -}; - -} // namespace internal - -// State is passed to a running Benchmark and contains state for the -// benchmark to use. -class BENCHMARK_EXPORT State { - public: - struct StateIterator; - friend struct StateIterator; - - // Returns iterators used to run each iteration of a benchmark using a - // C++11 ranged-based for loop. These functions should not be called directly. - // - // REQUIRES: The benchmark has not started running yet. Neither begin nor end - // have been called previously. - // - // NOTE: KeepRunning may not be used after calling either of these functions. - BENCHMARK_ALWAYS_INLINE StateIterator begin(); - BENCHMARK_ALWAYS_INLINE StateIterator end(); - - // Returns true if the benchmark should continue through another iteration. - // NOTE: A benchmark may not return from the test until KeepRunning() has - // returned false. - bool KeepRunning(); - - // Returns true iff the benchmark should run n more iterations. - // REQUIRES: 'n' > 0. - // NOTE: A benchmark must not return from the test until KeepRunningBatch() - // has returned false. - // NOTE: KeepRunningBatch() may overshoot by up to 'n' iterations. - // - // Intended usage: - // while (state.KeepRunningBatch(1000)) { - // // process 1000 elements - // } - bool KeepRunningBatch(IterationCount n); - - // REQUIRES: timer is running and 'SkipWithError(...)' has not been called - // by the current thread. - // Stop the benchmark timer. If not called, the timer will be - // automatically stopped after the last iteration of the benchmark loop. - // - // For threaded benchmarks the PauseTiming() function only pauses the timing - // for the current thread. - // - // NOTE: The "real time" measurement is per-thread. If different threads - // report different measurements the largest one is reported. - // - // NOTE: PauseTiming()/ResumeTiming() are relatively - // heavyweight, and so their use should generally be avoided - // within each benchmark iteration, if possible. - void PauseTiming(); - - // REQUIRES: timer is not running and 'SkipWithError(...)' has not been called - // by the current thread. - // Start the benchmark timer. The timer is NOT running on entrance to the - // benchmark function. It begins running after control flow enters the - // benchmark loop. - // - // NOTE: PauseTiming()/ResumeTiming() are relatively - // heavyweight, and so their use should generally be avoided - // within each benchmark iteration, if possible. - void ResumeTiming(); - - // REQUIRES: 'SkipWithError(...)' has not been called previously by the - // current thread. - // Report the benchmark as resulting in an error with the specified 'msg'. - // After this call the user may explicitly 'return' from the benchmark. - // - // If the ranged-for style of benchmark loop is used, the user must explicitly - // break from the loop, otherwise all future iterations will be run. - // If the 'KeepRunning()' loop is used the current thread will automatically - // exit the loop at the end of the current iteration. - // - // For threaded benchmarks only the current thread stops executing and future - // calls to `KeepRunning()` will block until all threads have completed - // the `KeepRunning()` loop. If multiple threads report an error only the - // first error message is used. - // - // NOTE: Calling 'SkipWithError(...)' does not cause the benchmark to exit - // the current scope immediately. If the function is called from within - // the 'KeepRunning()' loop the current iteration will finish. It is the users - // responsibility to exit the scope as needed. - void SkipWithError(const char* msg); - - // Returns true if an error has been reported with 'SkipWithError(...)'. - bool error_occurred() const { return error_occurred_; } - - // REQUIRES: called exactly once per iteration of the benchmarking loop. - // Set the manually measured time for this benchmark iteration, which - // is used instead of automatically measured time if UseManualTime() was - // specified. - // - // For threaded benchmarks the final value will be set to the largest - // reported values. - void SetIterationTime(double seconds); - - // Set the number of bytes processed by the current benchmark - // execution. This routine is typically called once at the end of a - // throughput oriented benchmark. - // - // REQUIRES: a benchmark has exited its benchmarking loop. - BENCHMARK_ALWAYS_INLINE - void SetBytesProcessed(int64_t bytes) { - counters["bytes_per_second"] = - Counter(static_cast(bytes), Counter::kIsRate, Counter::kIs1024); - } - - BENCHMARK_ALWAYS_INLINE - int64_t bytes_processed() const { - if (counters.find("bytes_per_second") != counters.end()) - return static_cast(counters.at("bytes_per_second")); - return 0; - } - - // If this routine is called with complexity_n > 0 and complexity report is - // requested for the - // family benchmark, then current benchmark will be part of the computation - // and complexity_n will - // represent the length of N. - BENCHMARK_ALWAYS_INLINE - void SetComplexityN(int64_t complexity_n) { complexity_n_ = complexity_n; } - - BENCHMARK_ALWAYS_INLINE - int64_t complexity_length_n() const { return complexity_n_; } - - // If this routine is called with items > 0, then an items/s - // label is printed on the benchmark report line for the currently - // executing benchmark. It is typically called at the end of a processing - // benchmark where a processing items/second output is desired. - // - // REQUIRES: a benchmark has exited its benchmarking loop. - BENCHMARK_ALWAYS_INLINE - void SetItemsProcessed(int64_t items) { - counters["items_per_second"] = - Counter(static_cast(items), benchmark::Counter::kIsRate); - } - - BENCHMARK_ALWAYS_INLINE - int64_t items_processed() const { - if (counters.find("items_per_second") != counters.end()) - return static_cast(counters.at("items_per_second")); - return 0; - } - - // If this routine is called, the specified label is printed at the - // end of the benchmark report line for the currently executing - // benchmark. Example: - // static void BM_Compress(benchmark::State& state) { - // ... - // double compress = input_size / output_size; - // state.SetLabel(StrFormat("compress:%.1f%%", 100.0*compression)); - // } - // Produces output that looks like: - // BM_Compress 50 50 14115038 compress:27.3% - // - // REQUIRES: a benchmark has exited its benchmarking loop. - void SetLabel(const char* label); - - void BENCHMARK_ALWAYS_INLINE SetLabel(const std::string& str) { - this->SetLabel(str.c_str()); - } - - // Range arguments for this run. CHECKs if the argument has been set. - BENCHMARK_ALWAYS_INLINE - int64_t range(std::size_t pos = 0) const { - assert(range_.size() > pos); - return range_[pos]; - } - - BENCHMARK_DEPRECATED_MSG("use 'range(0)' instead") - int64_t range_x() const { return range(0); } - - BENCHMARK_DEPRECATED_MSG("use 'range(1)' instead") - int64_t range_y() const { return range(1); } - - // Number of threads concurrently executing the benchmark. - BENCHMARK_ALWAYS_INLINE - int threads() const { return threads_; } - - // Index of the executing thread. Values from [0, threads). - BENCHMARK_ALWAYS_INLINE - int thread_index() const { return thread_index_; } - - BENCHMARK_ALWAYS_INLINE - IterationCount iterations() const { - if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) { - return 0; - } - return max_iterations - total_iterations_ + batch_leftover_; - } - - private: - // items we expect on the first cache line (ie 64 bytes of the struct) - // When total_iterations_ is 0, KeepRunning() and friends will return false. - // May be larger than max_iterations. - IterationCount total_iterations_; - - // When using KeepRunningBatch(), batch_leftover_ holds the number of - // iterations beyond max_iters that were run. Used to track - // completed_iterations_ accurately. - IterationCount batch_leftover_; - - public: - const IterationCount max_iterations; - - private: - bool started_; - bool finished_; - bool error_occurred_; - - // items we don't need on the first cache line - std::vector range_; - - int64_t complexity_n_; - - public: - // Container for user-defined counters. - UserCounters counters; - - private: - State(IterationCount max_iters, const std::vector& ranges, - int thread_i, int n_threads, internal::ThreadTimer* timer, - internal::ThreadManager* manager, - internal::PerfCountersMeasurement* perf_counters_measurement); - - void StartKeepRunning(); - // Implementation of KeepRunning() and KeepRunningBatch(). - // is_batch must be true unless n is 1. - bool KeepRunningInternal(IterationCount n, bool is_batch); - void FinishKeepRunning(); - - const int thread_index_; - const int threads_; - - internal::ThreadTimer* const timer_; - internal::ThreadManager* const manager_; - internal::PerfCountersMeasurement* const perf_counters_measurement_; - - friend class internal::BenchmarkInstance; -}; - -inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunning() { - return KeepRunningInternal(1, /*is_batch=*/false); -} - -inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningBatch(IterationCount n) { - return KeepRunningInternal(n, /*is_batch=*/true); -} - -inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(IterationCount n, - bool is_batch) { - // total_iterations_ is set to 0 by the constructor, and always set to a - // nonzero value by StartKepRunning(). - assert(n > 0); - // n must be 1 unless is_batch is true. - assert(is_batch || n == 1); - if (BENCHMARK_BUILTIN_EXPECT(total_iterations_ >= n, true)) { - total_iterations_ -= n; - return true; - } - if (!started_) { - StartKeepRunning(); - if (!error_occurred_ && total_iterations_ >= n) { - total_iterations_ -= n; - return true; - } - } - // For non-batch runs, total_iterations_ must be 0 by now. - if (is_batch && total_iterations_ != 0) { - batch_leftover_ = n - total_iterations_; - total_iterations_ = 0; - return true; - } - FinishKeepRunning(); - return false; -} - -struct State::StateIterator { - struct BENCHMARK_UNUSED Value {}; - typedef std::forward_iterator_tag iterator_category; - typedef Value value_type; - typedef Value reference; - typedef Value pointer; - typedef std::ptrdiff_t difference_type; - - private: - friend class State; - BENCHMARK_ALWAYS_INLINE - StateIterator() : cached_(0), parent_() {} - - BENCHMARK_ALWAYS_INLINE - explicit StateIterator(State* st) - : cached_(st->error_occurred_ ? 0 : st->max_iterations), parent_(st) {} - - public: - BENCHMARK_ALWAYS_INLINE - Value operator*() const { return Value(); } - - BENCHMARK_ALWAYS_INLINE - StateIterator& operator++() { - assert(cached_ > 0); - --cached_; - return *this; - } - - BENCHMARK_ALWAYS_INLINE - bool operator!=(StateIterator const&) const { - if (BENCHMARK_BUILTIN_EXPECT(cached_ != 0, true)) return true; - parent_->FinishKeepRunning(); - return false; - } - - private: - IterationCount cached_; - State* const parent_; -}; - -inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::begin() { - return StateIterator(this); -} -inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::end() { - StartKeepRunning(); - return StateIterator(); -} - -namespace internal { - -typedef void(Function)(State&); - -// ------------------------------------------------------ -// Benchmark registration object. The BENCHMARK() macro expands -// into an internal::Benchmark* object. Various methods can -// be called on this object to change the properties of the benchmark. -// Each method returns "this" so that multiple method calls can -// chained into one expression. -class BENCHMARK_EXPORT Benchmark { - public: - virtual ~Benchmark(); - - // Note: the following methods all return "this" so that multiple - // method calls can be chained together in one expression. - - // Specify the name of the benchmark - Benchmark* Name(const std::string& name); - - // Run this benchmark once with "x" as the extra argument passed - // to the function. - // REQUIRES: The function passed to the constructor must accept an arg1. - Benchmark* Arg(int64_t x); - - // Run this benchmark with the given time unit for the generated output report - Benchmark* Unit(TimeUnit unit); - - // Run this benchmark once for a number of values picked from the - // range [start..limit]. (start and limit are always picked.) - // REQUIRES: The function passed to the constructor must accept an arg1. - Benchmark* Range(int64_t start, int64_t limit); - - // Run this benchmark once for all values in the range [start..limit] with - // specific step - // REQUIRES: The function passed to the constructor must accept an arg1. - Benchmark* DenseRange(int64_t start, int64_t limit, int step = 1); - - // Run this benchmark once with "args" as the extra arguments passed - // to the function. - // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... - Benchmark* Args(const std::vector& args); - - // Equivalent to Args({x, y}) - // NOTE: This is a legacy C++03 interface provided for compatibility only. - // New code should use 'Args'. - Benchmark* ArgPair(int64_t x, int64_t y) { - std::vector args; - args.push_back(x); - args.push_back(y); - return Args(args); - } - - // Run this benchmark once for a number of values picked from the - // ranges [start..limit]. (starts and limits are always picked.) - // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... - Benchmark* Ranges(const std::vector >& ranges); - - // Run this benchmark once for each combination of values in the (cartesian) - // product of the supplied argument lists. - // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... - Benchmark* ArgsProduct(const std::vector >& arglists); - - // Equivalent to ArgNames({name}) - Benchmark* ArgName(const std::string& name); - - // Set the argument names to display in the benchmark name. If not called, - // only argument values will be shown. - Benchmark* ArgNames(const std::vector& names); - - // Equivalent to Ranges({{lo1, hi1}, {lo2, hi2}}). - // NOTE: This is a legacy C++03 interface provided for compatibility only. - // New code should use 'Ranges'. - Benchmark* RangePair(int64_t lo1, int64_t hi1, int64_t lo2, int64_t hi2) { - std::vector > ranges; - ranges.push_back(std::make_pair(lo1, hi1)); - ranges.push_back(std::make_pair(lo2, hi2)); - return Ranges(ranges); - } - - // Have "setup" and/or "teardown" invoked once for every benchmark run. - // If the benchmark is multi-threaded (will run in k threads concurrently), - // the setup callback will be be invoked exactly once (not k times) before - // each run with k threads. Time allowing (e.g. for a short benchmark), there - // may be multiple such runs per benchmark, each run with its own - // "setup"/"teardown". - // - // If the benchmark uses different size groups of threads (e.g. via - // ThreadRange), the above will be true for each size group. - // - // The callback will be passed a State object, which includes the number - // of threads, thread-index, benchmark arguments, etc. - // - // The callback must not be NULL or self-deleting. - Benchmark* Setup(void (*setup)(const benchmark::State&)); - Benchmark* Teardown(void (*teardown)(const benchmark::State&)); - - // Pass this benchmark object to *func, which can customize - // the benchmark by calling various methods like Arg, Args, - // Threads, etc. - Benchmark* Apply(void (*func)(Benchmark* benchmark)); - - // Set the range multiplier for non-dense range. If not called, the range - // multiplier kRangeMultiplier will be used. - Benchmark* RangeMultiplier(int multiplier); - - // Set the minimum amount of time to use when running this benchmark. This - // option overrides the `benchmark_min_time` flag. - // REQUIRES: `t > 0` and `Iterations` has not been called on this benchmark. - Benchmark* MinTime(double t); - - // Set the minimum amount of time to run the benchmark before taking runtimes - // of this benchmark into account. This - // option overrides the `benchmark_min_warmup_time` flag. - // REQUIRES: `t >= 0` and `Iterations` has not been called on this benchmark. - Benchmark* MinWarmUpTime(double t); - - // Specify the amount of iterations that should be run by this benchmark. - // REQUIRES: 'n > 0' and `MinTime` has not been called on this benchmark. - // - // NOTE: This function should only be used when *exact* iteration control is - // needed and never to control or limit how long a benchmark runs, where - // `--benchmark_min_time=N` or `MinTime(...)` should be used instead. - Benchmark* Iterations(IterationCount n); - - // Specify the amount of times to repeat this benchmark. This option overrides - // the `benchmark_repetitions` flag. - // REQUIRES: `n > 0` - Benchmark* Repetitions(int n); - - // Specify if each repetition of the benchmark should be reported separately - // or if only the final statistics should be reported. If the benchmark - // is not repeated then the single result is always reported. - // Applies to *ALL* reporters (display and file). - Benchmark* ReportAggregatesOnly(bool value = true); - - // Same as ReportAggregatesOnly(), but applies to display reporter only. - Benchmark* DisplayAggregatesOnly(bool value = true); - - // By default, the CPU time is measured only for the main thread, which may - // be unrepresentative if the benchmark uses threads internally. If called, - // the total CPU time spent by all the threads will be measured instead. - // By default, the only the main thread CPU time will be measured. - Benchmark* MeasureProcessCPUTime(); - - // If a particular benchmark should use the Wall clock instead of the CPU time - // (be it either the CPU time of the main thread only (default), or the - // total CPU usage of the benchmark), call this method. If called, the elapsed - // (wall) time will be used to control how many iterations are run, and in the - // printing of items/second or MB/seconds values. - // If not called, the CPU time used by the benchmark will be used. - Benchmark* UseRealTime(); - - // If a benchmark must measure time manually (e.g. if GPU execution time is - // being - // measured), call this method. If called, each benchmark iteration should - // call - // SetIterationTime(seconds) to report the measured time, which will be used - // to control how many iterations are run, and in the printing of items/second - // or MB/second values. - Benchmark* UseManualTime(); - - // Set the asymptotic computational complexity for the benchmark. If called - // the asymptotic computational complexity will be shown on the output. - Benchmark* Complexity(BigO complexity = benchmark::oAuto); - - // Set the asymptotic computational complexity for the benchmark. If called - // the asymptotic computational complexity will be shown on the output. - Benchmark* Complexity(BigOFunc* complexity); - - // Add this statistics to be computed over all the values of benchmark run - Benchmark* ComputeStatistics(const std::string& name, - StatisticsFunc* statistics, - StatisticUnit unit = kTime); - - // Support for running multiple copies of the same benchmark concurrently - // in multiple threads. This may be useful when measuring the scaling - // of some piece of code. - - // Run one instance of this benchmark concurrently in t threads. - Benchmark* Threads(int t); - - // Pick a set of values T from [min_threads,max_threads]. - // min_threads and max_threads are always included in T. Run this - // benchmark once for each value in T. The benchmark run for a - // particular value t consists of t threads running the benchmark - // function concurrently. For example, consider: - // BENCHMARK(Foo)->ThreadRange(1,16); - // This will run the following benchmarks: - // Foo in 1 thread - // Foo in 2 threads - // Foo in 4 threads - // Foo in 8 threads - // Foo in 16 threads - Benchmark* ThreadRange(int min_threads, int max_threads); - - // For each value n in the range, run this benchmark once using n threads. - // min_threads and max_threads are always included in the range. - // stride specifies the increment. E.g. DenseThreadRange(1, 8, 3) starts - // a benchmark with 1, 4, 7 and 8 threads. - Benchmark* DenseThreadRange(int min_threads, int max_threads, int stride = 1); - - // Equivalent to ThreadRange(NumCPUs(), NumCPUs()) - Benchmark* ThreadPerCpu(); - - virtual void Run(State& state) = 0; - - TimeUnit GetTimeUnit() const; - - protected: - explicit Benchmark(const char* name); - void SetName(const char* name); - - int ArgsCnt() const; - - private: - friend class BenchmarkFamilies; - friend class BenchmarkInstance; - - std::string name_; - AggregationReportMode aggregation_report_mode_; - std::vector arg_names_; // Args for all benchmark runs - std::vector > args_; // Args for all benchmark runs - - TimeUnit time_unit_; - bool use_default_time_unit_; - - int range_multiplier_; - double min_time_; - double min_warmup_time_; - IterationCount iterations_; - int repetitions_; - bool measure_process_cpu_time_; - bool use_real_time_; - bool use_manual_time_; - BigO complexity_; - BigOFunc* complexity_lambda_; - std::vector statistics_; - std::vector thread_counts_; - - typedef void (*callback_function)(const benchmark::State&); - callback_function setup_; - callback_function teardown_; - - Benchmark(Benchmark const&) -#if defined(BENCHMARK_HAS_CXX11) - = delete -#endif - ; - - Benchmark& operator=(Benchmark const&) -#if defined(BENCHMARK_HAS_CXX11) - = delete -#endif - ; -}; - -} // namespace internal - -// Create and register a benchmark with the specified 'name' that invokes -// the specified functor 'fn'. -// -// RETURNS: A pointer to the registered benchmark. -internal::Benchmark* RegisterBenchmark(const char* name, - internal::Function* fn); - -#if defined(BENCHMARK_HAS_CXX11) -template -internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn); -#endif - -// Remove all registered benchmarks. All pointers to previously registered -// benchmarks are invalidated. -BENCHMARK_EXPORT void ClearRegisteredBenchmarks(); - -namespace internal { -// The class used to hold all Benchmarks created from static function. -// (ie those created using the BENCHMARK(...) macros. -class BENCHMARK_EXPORT FunctionBenchmark : public Benchmark { - public: - FunctionBenchmark(const char* name, Function* func) - : Benchmark(name), func_(func) {} - - virtual void Run(State& st) BENCHMARK_OVERRIDE; - - private: - Function* func_; -}; - -#ifdef BENCHMARK_HAS_CXX11 -template -class LambdaBenchmark : public Benchmark { - public: - virtual void Run(State& st) BENCHMARK_OVERRIDE { lambda_(st); } - - private: - template - LambdaBenchmark(const char* name, OLambda&& lam) - : Benchmark(name), lambda_(std::forward(lam)) {} - - LambdaBenchmark(LambdaBenchmark const&) = delete; - - template // NOLINTNEXTLINE(readability-redundant-declaration) - friend Benchmark* ::benchmark::RegisterBenchmark(const char*, Lam&&); - - Lambda lambda_; -}; -#endif -} // namespace internal - -inline internal::Benchmark* RegisterBenchmark(const char* name, - internal::Function* fn) { - return internal::RegisterBenchmarkInternal( - ::new internal::FunctionBenchmark(name, fn)); -} - -#ifdef BENCHMARK_HAS_CXX11 -template -internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn) { - using BenchType = - internal::LambdaBenchmark::type>; - return internal::RegisterBenchmarkInternal( - ::new BenchType(name, std::forward(fn))); -} -#endif - -#if defined(BENCHMARK_HAS_CXX11) && \ - (!defined(BENCHMARK_GCC_VERSION) || BENCHMARK_GCC_VERSION >= 409) -template -internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn, - Args&&... args) { - return benchmark::RegisterBenchmark( - name, [=](benchmark::State& st) { fn(st, args...); }); -} -#else -#define BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK -#endif - -// The base class for all fixture tests. -class Fixture : public internal::Benchmark { - public: - Fixture() : internal::Benchmark("") {} - - virtual void Run(State& st) BENCHMARK_OVERRIDE { - this->SetUp(st); - this->BenchmarkCase(st); - this->TearDown(st); - } - - // These will be deprecated ... - virtual void SetUp(const State&) {} - virtual void TearDown(const State&) {} - // ... In favor of these. - virtual void SetUp(State& st) { SetUp(const_cast(st)); } - virtual void TearDown(State& st) { TearDown(const_cast(st)); } - - protected: - virtual void BenchmarkCase(State&) = 0; -}; -} // namespace benchmark - -// ------------------------------------------------------ -// Macro to register benchmarks - -// Check that __COUNTER__ is defined and that __COUNTER__ increases by 1 -// every time it is expanded. X + 1 == X + 0 is used in case X is defined to be -// empty. If X is empty the expression becomes (+1 == +0). -#if defined(__COUNTER__) && (__COUNTER__ + 1 == __COUNTER__ + 0) -#define BENCHMARK_PRIVATE_UNIQUE_ID __COUNTER__ -#else -#define BENCHMARK_PRIVATE_UNIQUE_ID __LINE__ -#endif - -// Helpers for generating unique variable names -#ifdef BENCHMARK_HAS_CXX11 -#define BENCHMARK_PRIVATE_NAME(...) \ - BENCHMARK_PRIVATE_CONCAT(benchmark_uniq_, BENCHMARK_PRIVATE_UNIQUE_ID, \ - __VA_ARGS__) -#else -#define BENCHMARK_PRIVATE_NAME(n) \ - BENCHMARK_PRIVATE_CONCAT(benchmark_uniq_, BENCHMARK_PRIVATE_UNIQUE_ID, n) -#endif // BENCHMARK_HAS_CXX11 - -#define BENCHMARK_PRIVATE_CONCAT(a, b, c) BENCHMARK_PRIVATE_CONCAT2(a, b, c) -#define BENCHMARK_PRIVATE_CONCAT2(a, b, c) a##b##c -// Helper for concatenation with macro name expansion -#define BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method) \ - BaseClass##_##Method##_Benchmark - -#define BENCHMARK_PRIVATE_DECLARE(n) \ - static ::benchmark::internal::Benchmark* BENCHMARK_PRIVATE_NAME(n) \ - BENCHMARK_UNUSED - -#ifdef BENCHMARK_HAS_CXX11 -#define BENCHMARK(...) \ - BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - new ::benchmark::internal::FunctionBenchmark(#__VA_ARGS__, \ - &__VA_ARGS__))) -#else -#define BENCHMARK(n) \ - BENCHMARK_PRIVATE_DECLARE(n) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - new ::benchmark::internal::FunctionBenchmark(#n, n))) -#endif // BENCHMARK_HAS_CXX11 - -// Old-style macros -#define BENCHMARK_WITH_ARG(n, a) BENCHMARK(n)->Arg((a)) -#define BENCHMARK_WITH_ARG2(n, a1, a2) BENCHMARK(n)->Args({(a1), (a2)}) -#define BENCHMARK_WITH_UNIT(n, t) BENCHMARK(n)->Unit((t)) -#define BENCHMARK_RANGE(n, lo, hi) BENCHMARK(n)->Range((lo), (hi)) -#define BENCHMARK_RANGE2(n, l1, h1, l2, h2) \ - BENCHMARK(n)->RangePair({{(l1), (h1)}, {(l2), (h2)}}) - -#ifdef BENCHMARK_HAS_CXX11 - -// Register a benchmark which invokes the function specified by `func` -// with the additional arguments specified by `...`. -// -// For example: -// -// template ` -// void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { -// [...] -//} -// /* Registers a benchmark named "BM_takes_args/int_string_test` */ -// BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); -#define BENCHMARK_CAPTURE(func, test_case_name, ...) \ - BENCHMARK_PRIVATE_DECLARE(func) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - new ::benchmark::internal::FunctionBenchmark( \ - #func "/" #test_case_name, \ - [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) - -#endif // BENCHMARK_HAS_CXX11 - -// This will register a benchmark for a templatized function. For example: -// -// template -// void BM_Foo(int iters); -// -// BENCHMARK_TEMPLATE(BM_Foo, 1); -// -// will register BM_Foo<1> as a benchmark. -#define BENCHMARK_TEMPLATE1(n, a) \ - BENCHMARK_PRIVATE_DECLARE(n) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - new ::benchmark::internal::FunctionBenchmark(#n "<" #a ">", n))) - -#define BENCHMARK_TEMPLATE2(n, a, b) \ - BENCHMARK_PRIVATE_DECLARE(n) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - new ::benchmark::internal::FunctionBenchmark(#n "<" #a "," #b ">", \ - n))) - -#ifdef BENCHMARK_HAS_CXX11 -#define BENCHMARK_TEMPLATE(n, ...) \ - BENCHMARK_PRIVATE_DECLARE(n) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - new ::benchmark::internal::FunctionBenchmark( \ - #n "<" #__VA_ARGS__ ">", n<__VA_ARGS__>))) -#else -#define BENCHMARK_TEMPLATE(n, a) BENCHMARK_TEMPLATE1(n, a) -#endif - -#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "/" #Method); \ - } \ - \ - protected: \ - virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ - }; - -#define BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "<" #a ">/" #Method); \ - } \ - \ - protected: \ - virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ - }; - -#define BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "<" #a "," #b ">/" #Method); \ - } \ - \ - protected: \ - virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ - }; - -#ifdef BENCHMARK_HAS_CXX11 -#define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, ...) \ - class BaseClass##_##Method##_Benchmark : public BaseClass<__VA_ARGS__> { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "<" #__VA_ARGS__ ">/" #Method); \ - } \ - \ - protected: \ - virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ - }; -#else -#define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(n, a) \ - BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(n, a) -#endif - -#define BENCHMARK_DEFINE_F(BaseClass, Method) \ - BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ - void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase - -#define BENCHMARK_TEMPLATE1_DEFINE_F(BaseClass, Method, a) \ - BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ - void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase - -#define BENCHMARK_TEMPLATE2_DEFINE_F(BaseClass, Method, a, b) \ - BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ - void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase - -#ifdef BENCHMARK_HAS_CXX11 -#define BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, ...) \ - BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, __VA_ARGS__) \ - void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase -#else -#define BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, a) \ - BENCHMARK_TEMPLATE1_DEFINE_F(BaseClass, Method, a) -#endif - -#define BENCHMARK_REGISTER_F(BaseClass, Method) \ - BENCHMARK_PRIVATE_REGISTER_F(BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)) - -#define BENCHMARK_PRIVATE_REGISTER_F(TestName) \ - BENCHMARK_PRIVATE_DECLARE(TestName) = \ - (::benchmark::internal::RegisterBenchmarkInternal(new TestName())) - -// This macro will define and register a benchmark within a fixture class. -#define BENCHMARK_F(BaseClass, Method) \ - BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ - BENCHMARK_REGISTER_F(BaseClass, Method); \ - void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase - -#define BENCHMARK_TEMPLATE1_F(BaseClass, Method, a) \ - BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ - BENCHMARK_REGISTER_F(BaseClass, Method); \ - void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase - -#define BENCHMARK_TEMPLATE2_F(BaseClass, Method, a, b) \ - BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ - BENCHMARK_REGISTER_F(BaseClass, Method); \ - void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase - -#ifdef BENCHMARK_HAS_CXX11 -#define BENCHMARK_TEMPLATE_F(BaseClass, Method, ...) \ - BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, __VA_ARGS__) \ - BENCHMARK_REGISTER_F(BaseClass, Method); \ - void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase -#else -#define BENCHMARK_TEMPLATE_F(BaseClass, Method, a) \ - BENCHMARK_TEMPLATE1_F(BaseClass, Method, a) -#endif - -// Helper macro to create a main routine in a test that runs the benchmarks -#define BENCHMARK_MAIN() \ - int main(int argc, char** argv) { \ - ::benchmark::Initialize(&argc, argv); \ - if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1; \ - ::benchmark::RunSpecifiedBenchmarks(); \ - ::benchmark::Shutdown(); \ - return 0; \ - } \ - int main(int, char**) - -// ------------------------------------------------------ -// Benchmark Reporters - -namespace benchmark { - -struct BENCHMARK_EXPORT CPUInfo { - struct CacheInfo { - std::string type; - int level; - int size; - int num_sharing; - }; - - enum Scaling { UNKNOWN, ENABLED, DISABLED }; - - int num_cpus; - Scaling scaling; - double cycles_per_second; - std::vector caches; - std::vector load_avg; - - static const CPUInfo& Get(); - - private: - CPUInfo(); - BENCHMARK_DISALLOW_COPY_AND_ASSIGN(CPUInfo); -}; - -// Adding Struct for System Information -struct BENCHMARK_EXPORT SystemInfo { - std::string name; - static const SystemInfo& Get(); - - private: - SystemInfo(); - BENCHMARK_DISALLOW_COPY_AND_ASSIGN(SystemInfo); -}; - -// BenchmarkName contains the components of the Benchmark's name -// which allows individual fields to be modified or cleared before -// building the final name using 'str()'. -struct BENCHMARK_EXPORT BenchmarkName { - std::string function_name; - std::string args; - std::string min_time; - std::string min_warmup_time; - std::string iterations; - std::string repetitions; - std::string time_type; - std::string threads; - - // Return the full name of the benchmark with each non-empty - // field separated by a '/' - std::string str() const; -}; - -// Interface for custom benchmark result printers. -// By default, benchmark reports are printed to stdout. However an application -// can control the destination of the reports by calling -// RunSpecifiedBenchmarks and passing it a custom reporter object. -// The reporter object must implement the following interface. -class BENCHMARK_EXPORT BenchmarkReporter { - public: - struct Context { - CPUInfo const& cpu_info; - SystemInfo const& sys_info; - // The number of chars in the longest benchmark name. - size_t name_field_width; - static const char* executable_name; - Context(); - }; - - struct BENCHMARK_EXPORT Run { - static const int64_t no_repetition_index = -1; - enum RunType { RT_Iteration, RT_Aggregate }; - - Run() - : run_type(RT_Iteration), - aggregate_unit(kTime), - error_occurred(false), - iterations(1), - threads(1), - time_unit(GetDefaultTimeUnit()), - real_accumulated_time(0), - cpu_accumulated_time(0), - max_heapbytes_used(0), - complexity(oNone), - complexity_lambda(), - complexity_n(0), - report_big_o(false), - report_rms(false), - memory_result(NULL), - allocs_per_iter(0.0) {} - - std::string benchmark_name() const; - BenchmarkName run_name; - int64_t family_index; - int64_t per_family_instance_index; - RunType run_type; - std::string aggregate_name; - StatisticUnit aggregate_unit; - std::string report_label; // Empty if not set by benchmark. - bool error_occurred; - std::string error_message; - - IterationCount iterations; - int64_t threads; - int64_t repetition_index; - int64_t repetitions; - TimeUnit time_unit; - double real_accumulated_time; - double cpu_accumulated_time; - - // Return a value representing the real time per iteration in the unit - // specified by 'time_unit'. - // NOTE: If 'iterations' is zero the returned value represents the - // accumulated time. - double GetAdjustedRealTime() const; - - // Return a value representing the cpu time per iteration in the unit - // specified by 'time_unit'. - // NOTE: If 'iterations' is zero the returned value represents the - // accumulated time. - double GetAdjustedCPUTime() const; - - // This is set to 0.0 if memory tracing is not enabled. - double max_heapbytes_used; - - // Keep track of arguments to compute asymptotic complexity - BigO complexity; - BigOFunc* complexity_lambda; - int64_t complexity_n; - - // what statistics to compute from the measurements - const std::vector* statistics; - - // Inform print function whether the current run is a complexity report - bool report_big_o; - bool report_rms; - - UserCounters counters; - - // Memory metrics. - const MemoryManager::Result* memory_result; - double allocs_per_iter; - }; - - struct PerFamilyRunReports { - PerFamilyRunReports() : num_runs_total(0), num_runs_done(0) {} - - // How many runs will all instances of this benchmark perform? - int num_runs_total; - - // How many runs have happened already? - int num_runs_done; - - // The reports about (non-errneous!) runs of this family. - std::vector Runs; - }; - - // Construct a BenchmarkReporter with the output stream set to 'std::cout' - // and the error stream set to 'std::cerr' - BenchmarkReporter(); - - // Called once for every suite of benchmarks run. - // The parameter "context" contains information that the - // reporter may wish to use when generating its report, for example the - // platform under which the benchmarks are running. The benchmark run is - // never started if this function returns false, allowing the reporter - // to skip runs based on the context information. - virtual bool ReportContext(const Context& context) = 0; - - // Called once for each group of benchmark runs, gives information about - // cpu-time and heap memory usage during the benchmark run. If the group - // of runs contained more than two entries then 'report' contains additional - // elements representing the mean and standard deviation of those runs. - // Additionally if this group of runs was the last in a family of benchmarks - // 'reports' contains additional entries representing the asymptotic - // complexity and RMS of that benchmark family. - virtual void ReportRuns(const std::vector& report) = 0; - - // Called once and only once after ever group of benchmarks is run and - // reported. - virtual void Finalize() {} - - // REQUIRES: The object referenced by 'out' is valid for the lifetime - // of the reporter. - void SetOutputStream(std::ostream* out) { - assert(out); - output_stream_ = out; - } - - // REQUIRES: The object referenced by 'err' is valid for the lifetime - // of the reporter. - void SetErrorStream(std::ostream* err) { - assert(err); - error_stream_ = err; - } - - std::ostream& GetOutputStream() const { return *output_stream_; } - - std::ostream& GetErrorStream() const { return *error_stream_; } - - virtual ~BenchmarkReporter(); - - // Write a human readable string to 'out' representing the specified - // 'context'. - // REQUIRES: 'out' is non-null. - static void PrintBasicContext(std::ostream* out, Context const& context); - - private: - std::ostream* output_stream_; - std::ostream* error_stream_; -}; - -// Simple reporter that outputs benchmark data to the console. This is the -// default reporter used by RunSpecifiedBenchmarks(). -class BENCHMARK_EXPORT ConsoleReporter : public BenchmarkReporter { - public: - enum OutputOptions { - OO_None = 0, - OO_Color = 1, - OO_Tabular = 2, - OO_ColorTabular = OO_Color | OO_Tabular, - OO_Defaults = OO_ColorTabular - }; - explicit ConsoleReporter(OutputOptions opts_ = OO_Defaults) - : output_options_(opts_), name_field_width_(0), printed_header_(false) {} - - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; - virtual void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; - - protected: - virtual void PrintRunData(const Run& report); - virtual void PrintHeader(const Run& report); - - OutputOptions output_options_; - size_t name_field_width_; - UserCounters prev_counters_; - bool printed_header_; -}; - -class BENCHMARK_EXPORT JSONReporter : public BenchmarkReporter { - public: - JSONReporter() : first_report_(true) {} - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; - virtual void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; - virtual void Finalize() BENCHMARK_OVERRIDE; - - private: - void PrintRunData(const Run& report); - - bool first_report_; -}; - -class BENCHMARK_EXPORT BENCHMARK_DEPRECATED_MSG( - "The CSV Reporter will be removed in a future release") CSVReporter - : public BenchmarkReporter { - public: - CSVReporter() : printed_header_(false) {} - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; - virtual void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; - - private: - void PrintRunData(const Run& report); - - bool printed_header_; - std::set user_counter_names_; -}; - -inline const char* GetTimeUnitString(TimeUnit unit) { - switch (unit) { - case kSecond: - return "s"; - case kMillisecond: - return "ms"; - case kMicrosecond: - return "us"; - case kNanosecond: - return "ns"; - } - BENCHMARK_UNREACHABLE(); -} - -inline double GetTimeUnitMultiplier(TimeUnit unit) { - switch (unit) { - case kSecond: - return 1; - case kMillisecond: - return 1e3; - case kMicrosecond: - return 1e6; - case kNanosecond: - return 1e9; - } - BENCHMARK_UNREACHABLE(); -} - -// Creates a list of integer values for the given range and multiplier. -// This can be used together with ArgsProduct() to allow multiple ranges -// with different multiplers. -// Example: -// ArgsProduct({ -// CreateRange(0, 1024, /*multi=*/32), -// CreateRange(0, 100, /*multi=*/4), -// CreateDenseRange(0, 4, /*step=*/1), -// }); -BENCHMARK_EXPORT -std::vector CreateRange(int64_t lo, int64_t hi, int multi); - -// Creates a list of integer values for the given range and step. -BENCHMARK_EXPORT -std::vector CreateDenseRange(int64_t start, int64_t limit, int step); - -} // namespace benchmark - -#if defined(_MSC_VER) -#pragma warning(pop) -#endif +#include "benchmark/benchmark_api.h" +#include "benchmark/counter.h" +#include "benchmark/macros.h" +#include "benchmark/managers.h" +#include "benchmark/registration.h" +#include "benchmark/reporter.h" +#include "benchmark/state.h" +#include "benchmark/statistics.h" +#include "benchmark/sysinfo.h" +#include "benchmark/types.h" +#include "benchmark/utils.h" #endif // BENCHMARK_BENCHMARK_H_ diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h new file mode 100644 index 0000000000..6d98aba581 --- /dev/null +++ b/include/benchmark/benchmark_api.h @@ -0,0 +1,292 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BENCHMARK_BENCHMARK_API_H_ +#define BENCHMARK_BENCHMARK_API_H_ + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4251) +#endif + +#include +#include +#include +#include +#include + +#include "benchmark/counter.h" +#include "benchmark/macros.h" +#include "benchmark/state.h" +#include "benchmark/statistics.h" +#include "benchmark/types.h" + +namespace benchmark { + +const char kDefaultMinTimeStr[] = "0.5s"; + +BENCHMARK_EXPORT void MaybeReenterWithoutASLR(int, char**); + +BENCHMARK_EXPORT std::string GetBenchmarkVersion(); + +BENCHMARK_EXPORT void PrintDefaultHelp(); + +BENCHMARK_EXPORT void Initialize(int* argc, char** argv, + void (*HelperPrintf)() = PrintDefaultHelp); +BENCHMARK_EXPORT void Shutdown(); + +BENCHMARK_EXPORT bool ReportUnrecognizedArguments(int argc, char** argv); + +BENCHMARK_EXPORT std::string GetBenchmarkFilter(); + +BENCHMARK_EXPORT void SetBenchmarkFilter(std::string value); + +BENCHMARK_EXPORT int32_t GetBenchmarkVerbosity(); + +BENCHMARK_EXPORT BenchmarkReporter* CreateDefaultDisplayReporter(); + +BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks(); +BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks(std::string spec); + +BENCHMARK_EXPORT size_t +RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter); +BENCHMARK_EXPORT size_t +RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, std::string spec); + +BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks( + BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter); +BENCHMARK_EXPORT size_t +RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, + BenchmarkReporter* file_reporter, std::string spec); + +BENCHMARK_EXPORT TimeUnit GetDefaultTimeUnit(); + +BENCHMARK_EXPORT void SetDefaultTimeUnit(TimeUnit unit); + +BENCHMARK_EXPORT +void AddCustomContext(std::string key, std::string value); + +struct ThreadRunnerBase { + virtual ~ThreadRunnerBase() {} + virtual void RunThreads(const std::function& fn) = 0; +}; + +using threadrunner_factory = + std::function(int)>; + +namespace internal { +class BenchmarkFamilies; +class BenchmarkInstance; +} // namespace internal + +class BENCHMARK_EXPORT Benchmark { + public: + virtual ~Benchmark(); + + Benchmark* Name(const std::string& name); + Benchmark* Arg(int64_t x); + Benchmark* Unit(TimeUnit unit); + Benchmark* Range(int64_t start, int64_t limit); + Benchmark* DenseRange(int64_t start, int64_t limit, int step = 1); + Benchmark* Args(const std::vector& args); + Benchmark* ArgPair(int64_t x, int64_t y) { + std::vector args; + args.push_back(x); + args.push_back(y); + return Args(args); + } + Benchmark* Ranges(const std::vector>& ranges); + Benchmark* ArgsProduct(const std::vector>& arglists); + Benchmark* ArgName(const std::string& name); + Benchmark* ArgNames(const std::vector& names); + Benchmark* RangePair(int64_t lo1, int64_t hi1, int64_t lo2, int64_t hi2) { + std::vector> ranges; + ranges.push_back(std::make_pair(lo1, hi1)); + ranges.push_back(std::make_pair(lo2, hi2)); + return Ranges(ranges); + } + Benchmark* Setup(callback_function&&); + Benchmark* Setup(const callback_function&); + Benchmark* Teardown(callback_function&&); + Benchmark* Teardown(const callback_function&); + Benchmark* Apply(const std::function&); + Benchmark* RangeMultiplier(int multiplier); + Benchmark* MinTime(double t); + Benchmark* MinWarmUpTime(double t); + Benchmark* Iterations(IterationCount n); + Benchmark* Repetitions(int n); + Benchmark* ReportAggregatesOnly(bool value = true); + Benchmark* DisplayAggregatesOnly(bool value = true); + Benchmark* MeasureProcessCPUTime(); + Benchmark* UseRealTime(); + Benchmark* UseManualTime(); + Benchmark* Complexity(BigO complexity = benchmark::oAuto); + Benchmark* Complexity(BigOFunc* complexity); + Benchmark* ComputeStatistics(const std::string& name, + StatisticsFunc* statistics, + StatisticUnit unit = kTime); + Benchmark* Threads(int t); + Benchmark* ThreadRange(int min_threads, int max_threads); + Benchmark* DenseThreadRange(int min_threads, int max_threads, int stride = 1); + Benchmark* ThreadPerCpu(); + Benchmark* ThreadRunner(threadrunner_factory&& factory); + + virtual void Run(State& state) = 0; + + TimeUnit GetTimeUnit() const; + + protected: + explicit Benchmark(const std::string& name); + void SetName(const std::string& name); + + public: + const char* GetName() const; + int ArgsCnt() const; + const char* GetArgName(int arg) const; + + private: + friend class internal::BenchmarkFamilies; + friend class internal::BenchmarkInstance; + + std::string name_; + internal::AggregationReportMode aggregation_report_mode_; + std::vector arg_names_; + std::vector> args_; + + TimeUnit time_unit_; + bool use_default_time_unit_; + + int range_multiplier_; + double min_time_; + double min_warmup_time_; + IterationCount iterations_; + int repetitions_; + bool measure_process_cpu_time_; + bool use_real_time_; + bool use_manual_time_; + BigO complexity_; + BigOFunc* complexity_lambda_; + std::vector statistics_; + std::vector thread_counts_; + + callback_function setup_; + callback_function teardown_; + + threadrunner_factory threadrunner_; + + BENCHMARK_DISALLOW_COPY_AND_ASSIGN(Benchmark); +}; + +namespace internal { +typedef BENCHMARK_DEPRECATED_MSG( + "Use ::benchmark::Benchmark instead")::benchmark::Benchmark Benchmark; +typedef BENCHMARK_DEPRECATED_MSG( + "Use ::benchmark::threadrunner_factory instead")::benchmark:: + threadrunner_factory threadrunner_factory; + +typedef void(Function)(State&); + +BENCHMARK_EXPORT ::benchmark::Benchmark* RegisterBenchmarkInternal( + std::unique_ptr<::benchmark::Benchmark>); +BENCHMARK_EXPORT std::map*& GetGlobalContext(); +BENCHMARK_EXPORT void UseCharPointer(char const volatile*); +BENCHMARK_EXPORT int InitializeStreams(); +BENCHMARK_UNUSED static int stream_init_anchor = InitializeStreams(); +} // namespace internal + +Benchmark* RegisterBenchmark(const std::string& name, internal::Function* fn); + +template +Benchmark* RegisterBenchmark(const std::string& name, Lambda&& fn); + +BENCHMARK_EXPORT void ClearRegisteredBenchmarks(); + +namespace internal { +class BENCHMARK_EXPORT FunctionBenchmark : public benchmark::Benchmark { + public: + FunctionBenchmark(const std::string& name, Function* func) + : Benchmark(name), func_(func) {} + void Run(State& st) override; + + private: + Function* func_; +}; + +template +class LambdaBenchmark : public benchmark::Benchmark { + public: + void Run(State& st) override { lambda_(st); } + template + LambdaBenchmark(const std::string& name, OLambda&& lam) + : Benchmark(name), lambda_(std::forward(lam)) {} + + private: + LambdaBenchmark(LambdaBenchmark const&) = delete; + Lambda lambda_; +}; +} // namespace internal + +inline Benchmark* RegisterBenchmark(const std::string& name, + internal::Function* fn) { + return internal::RegisterBenchmarkInternal( + ::benchmark::internal::make_unique(name, + fn)); +} + +template +Benchmark* RegisterBenchmark(const std::string& name, Lambda&& fn) { + using BenchType = + internal::LambdaBenchmark::type>; + return internal::RegisterBenchmarkInternal( + ::benchmark::internal::make_unique(name, + std::forward(fn))); +} + +template +Benchmark* RegisterBenchmark(const std::string& name, Lambda&& fn, + Args&&... args) { + return benchmark::RegisterBenchmark( + name, [=](benchmark::State& st) { fn(st, args...); }); +} + +class Fixture : public Benchmark { + public: + Fixture() : Benchmark("") {} + void Run(State& st) override { + this->SetUp(st); + this->BenchmarkCase(st); + this->TearDown(st); + } + virtual void SetUp(const State&) {} + virtual void TearDown(const State&) {} + virtual void SetUp(State& st) { SetUp(const_cast(st)); } + virtual void TearDown(State& st) { TearDown(const_cast(st)); } + + protected: + virtual void BenchmarkCase(State&) = 0; +}; + +BENCHMARK_EXPORT +std::vector CreateRange(int64_t lo, int64_t hi, int multi); + +BENCHMARK_EXPORT +std::vector CreateDenseRange(int64_t start, int64_t limit, int step); + +} // namespace benchmark + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +#endif // BENCHMARK_BENCHMARK_API_H_ diff --git a/include/benchmark/counter.h b/include/benchmark/counter.h new file mode 100644 index 0000000000..8db5d8dad7 --- /dev/null +++ b/include/benchmark/counter.h @@ -0,0 +1,80 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BENCHMARK_COUNTER_H_ +#define BENCHMARK_COUNTER_H_ + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4251) +#endif + +#include +#include + +#include "benchmark/macros.h" +#include "benchmark/types.h" + +namespace benchmark { + +class BENCHMARK_EXPORT Counter { + public: + enum Flags { + kDefaults = 0, + kIsRate = 1 << 0, + kAvgThreads = 1 << 1, + kAvgThreadsRate = kIsRate | kAvgThreads, + kIsIterationInvariant = 1 << 2, + kIsIterationInvariantRate = kIsRate | kIsIterationInvariant, + kAvgIterations = 1 << 3, + kAvgIterationsRate = kIsRate | kAvgIterations, + kInvert = 1 << 31 + }; + + enum OneK { kIs1000 = 1000, kIs1024 = 1024 }; + + double value; + Flags flags; + OneK oneK; + + BENCHMARK_ALWAYS_INLINE + Counter(double v = 0., Flags f = kDefaults, OneK k = kIs1000) + : value(v), flags(f), oneK(k) {} + + BENCHMARK_ALWAYS_INLINE operator double const&() const { return value; } + BENCHMARK_ALWAYS_INLINE operator double&() { return value; } +}; + +Counter::Flags inline operator|(const Counter::Flags& LHS, + const Counter::Flags& RHS) { + return static_cast(static_cast(LHS) | + static_cast(RHS)); +} + +using UserCounters = std::map; + +namespace internal { +void Finish(UserCounters* l, IterationCount iterations, double cpu_time, + double num_threads); +void Increment(UserCounters* l, UserCounters const& r); +bool SameNames(UserCounters const& l, UserCounters const& r); +} // namespace internal + +} // namespace benchmark + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +#endif // BENCHMARK_COUNTER_H_ diff --git a/include/benchmark/export.h b/include/benchmark/export.h new file mode 100644 index 0000000000..f96f8596cd --- /dev/null +++ b/include/benchmark/export.h @@ -0,0 +1,47 @@ +#ifndef BENCHMARK_EXPORT_H +#define BENCHMARK_EXPORT_H + +#if defined(_WIN32) +#define EXPORT_ATTR __declspec(dllexport) +#define IMPORT_ATTR __declspec(dllimport) +#define NO_EXPORT_ATTR +#define DEPRECATED_ATTR __declspec(deprecated) +#else // _WIN32 +#define EXPORT_ATTR __attribute__((visibility("default"))) +#define IMPORT_ATTR __attribute__((visibility("default"))) +#define NO_EXPORT_ATTR __attribute__((visibility("hidden"))) +#define DEPRECATE_ATTR __attribute__((__deprecated__)) +#endif // _WIN32 + +#ifdef BENCHMARK_STATIC_DEFINE +#define BENCHMARK_EXPORT +#define BENCHMARK_NO_EXPORT +#else // BENCHMARK_STATIC_DEFINE +#ifndef BENCHMARK_EXPORT +#ifdef benchmark_EXPORTS +/* We are building this library */ +#define BENCHMARK_EXPORT EXPORT_ATTR +#else // benchmark_EXPORTS +/* We are using this library */ +#define BENCHMARK_EXPORT IMPORT_ATTR +#endif // benchmark_EXPORTS +#endif // !BENCHMARK_EXPORT + +#ifndef BENCHMARK_NO_EXPORT +#define BENCHMARK_NO_EXPORT NO_EXPORT_ATTR +#endif // !BENCHMARK_NO_EXPORT +#endif // BENCHMARK_STATIC_DEFINE + +#ifndef BENCHMARK_DEPRECATED +#define BENCHMARK_DEPRECATED DEPRECATE_ATTR +#endif // BENCHMARK_DEPRECATED + +#ifndef BENCHMARK_DEPRECATED_EXPORT +#define BENCHMARK_DEPRECATED_EXPORT BENCHMARK_EXPORT BENCHMARK_DEPRECATED +#endif // BENCHMARK_DEPRECATED_EXPORT + +#ifndef BENCHMARK_DEPRECATED_NO_EXPORT +#define BENCHMARK_DEPRECATED_NO_EXPORT BENCHMARK_NO_EXPORT BENCHMARK_DEPRECATED +#endif // BENCHMARK_DEPRECATED_EXPORT + +#endif /* BENCHMARK_EXPORT_H */ diff --git a/include/benchmark/macros.h b/include/benchmark/macros.h new file mode 100644 index 0000000000..17f19ac272 --- /dev/null +++ b/include/benchmark/macros.h @@ -0,0 +1,136 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BENCHMARK_MACROS_H_ +#define BENCHMARK_MACROS_H_ + +#if defined(_MSC_VER) +#include +#endif + +#include + +#define BENCHMARK_DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&) = delete; \ + TypeName& operator=(const TypeName&) = delete + +#ifdef BENCHMARK_HAS_CXX17 +#define BENCHMARK_UNUSED [[maybe_unused]] +#elif defined(__GNUC__) || defined(__clang__) +#define BENCHMARK_UNUSED __attribute__((unused)) +#else +#define BENCHMARK_UNUSED +#endif + +#if defined(__clang__) +#define BENCHMARK_DONT_OPTIMIZE __attribute__((optnone)) +#elif defined(__GNUC__) || defined(__GNUG__) +#define BENCHMARK_DONT_OPTIMIZE __attribute__((optimize(0))) +#else +#define BENCHMARK_DONT_OPTIMIZE +#endif + +#if defined(__GNUC__) || defined(__clang__) +#define BENCHMARK_ALWAYS_INLINE __attribute__((always_inline)) +#elif defined(_MSC_VER) && !defined(__clang__) +#define BENCHMARK_ALWAYS_INLINE __forceinline +#define __func__ __FUNCTION__ +#else +#define BENCHMARK_ALWAYS_INLINE +#endif + +#define BENCHMARK_INTERNAL_TOSTRING2(x) #x +#define BENCHMARK_INTERNAL_TOSTRING(x) BENCHMARK_INTERNAL_TOSTRING2(x) + +#if (defined(__GNUC__) && !defined(__NVCC__) && !defined(__NVCOMPILER)) || \ + defined(__clang__) +#define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y) +#define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) +#define BENCHMARK_DISABLE_DEPRECATED_WARNING \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#define BENCHMARK_RESTORE_DEPRECATED_WARNING _Pragma("GCC diagnostic pop") +#elif defined(__NVCOMPILER) +#define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y) +#define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) +#define BENCHMARK_DISABLE_DEPRECATED_WARNING \ + _Pragma("diagnostic push") \ + _Pragma("diag_suppress deprecated_entity_with_custom_message") +#define BENCHMARK_RESTORE_DEPRECATED_WARNING _Pragma("diagnostic pop") +#elif defined(_MSC_VER) +#define BENCHMARK_BUILTIN_EXPECT(x, y) x +#define BENCHMARK_DEPRECATED_MSG(msg) __declspec(deprecated(msg)) +#define BENCHMARK_WARNING_MSG(msg) \ + __pragma(message(__FILE__ "(" BENCHMARK_INTERNAL_TOSTRING( \ + __LINE__) ") : warning note: " msg)) +#define BENCHMARK_DISABLE_DEPRECATED_WARNING \ + __pragma(warning(push)) __pragma(warning(disable : 4996)) +#define BENCHMARK_RESTORE_DEPRECATED_WARNING __pragma(warning(pop)) +#else +#define BENCHMARK_BUILTIN_EXPECT(x, y) x +#define BENCHMARK_DEPRECATED_MSG(msg) +#define BENCHMARK_WARNING_MSG(msg) \ + __pragma(message(__FILE__ "(" BENCHMARK_INTERNAL_TOSTRING( \ + __LINE__) ") : warning note: " msg)) +#define BENCHMARK_DISABLE_DEPRECATED_WARNING +#define BENCHMARK_RESTORE_DEPRECATED_WARNING +#endif + +#if defined(__GNUC__) && !defined(__clang__) +#define BENCHMARK_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#endif + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +#if defined(__GNUC__) || __has_builtin(__builtin_unreachable) +#define BENCHMARK_UNREACHABLE() __builtin_unreachable() +#elif defined(_MSC_VER) +#define BENCHMARK_UNREACHABLE() __assume(false) +#else +#define BENCHMARK_UNREACHABLE() ((void)0) +#endif + +#if defined(__GNUC__) +#if defined(__i386__) || defined(__x86_64__) +#define BENCHMARK_INTERNAL_CACHELINE_SIZE 64 +#elif defined(__powerpc64__) +#define BENCHMARK_INTERNAL_CACHELINE_SIZE 128 +#elif defined(__aarch64__) +#define BENCHMARK_INTERNAL_CACHELINE_SIZE 64 +#elif defined(__arm__) +#if defined(__ARM_ARCH_5T__) +#define BENCHMARK_INTERNAL_CACHELINE_SIZE 32 +#elif defined(__ARM_ARCH_7A__) +#define BENCHMARK_INTERNAL_CACHELINE_SIZE 64 +#endif +#endif +#endif + +#ifndef BENCHMARK_INTERNAL_CACHELINE_SIZE +#define BENCHMARK_INTERNAL_CACHELINE_SIZE 64 +#endif + +#if defined(__GNUC__) +#define BENCHMARK_INTERNAL_CACHELINE_ALIGNED \ + __attribute__((aligned(BENCHMARK_INTERNAL_CACHELINE_SIZE))) +#elif defined(_MSC_VER) +#define BENCHMARK_INTERNAL_CACHELINE_ALIGNED \ + __declspec(align(BENCHMARK_INTERNAL_CACHELINE_SIZE)) +#else +#define BENCHMARK_INTERNAL_CACHELINE_ALIGNED +#endif + +#endif // BENCHMARK_MACROS_H_ diff --git a/include/benchmark/managers.h b/include/benchmark/managers.h new file mode 100644 index 0000000000..e8b6cd4749 --- /dev/null +++ b/include/benchmark/managers.h @@ -0,0 +1,66 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BENCHMARK_MANAGERS_H_ +#define BENCHMARK_MANAGERS_H_ + +#include + +#include + +#include "benchmark/macros.h" +#include "benchmark/types.h" + +namespace benchmark { + +class MemoryManager { + public: + static constexpr int64_t TombstoneValue = std::numeric_limits::max(); + + struct Result { + Result() + : num_allocs(0), + max_bytes_used(0), + total_allocated_bytes(TombstoneValue), + net_heap_growth(TombstoneValue), + memory_iterations(0) {} + + int64_t num_allocs; + int64_t max_bytes_used; + int64_t total_allocated_bytes; + int64_t net_heap_growth; + IterationCount memory_iterations; + }; + + virtual ~MemoryManager() {} + virtual void Start() = 0; + virtual void Stop(Result& result) = 0; +}; + +BENCHMARK_EXPORT +void RegisterMemoryManager(MemoryManager* memory_manager); + +class ProfilerManager { + public: + virtual ~ProfilerManager() {} + virtual void AfterSetupStart() = 0; + virtual void BeforeTeardownStop() = 0; +}; + +BENCHMARK_EXPORT +void RegisterProfilerManager(ProfilerManager* profiler_manager); + +} // namespace benchmark + +#endif // BENCHMARK_MANAGERS_H_ diff --git a/include/benchmark/registration.h b/include/benchmark/registration.h new file mode 100644 index 0000000000..5ac08de3fc --- /dev/null +++ b/include/benchmark/registration.h @@ -0,0 +1,258 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BENCHMARK_REGISTRATION_H_ +#define BENCHMARK_REGISTRATION_H_ + +#include "benchmark/benchmark_api.h" +#include "benchmark/macros.h" + +#if defined(__clang__) +#define BENCHMARK_DISABLE_COUNTER_WARNING \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wunknown-warning-option\"") \ + _Pragma("GCC diagnostic ignored \"-Wc2y-extensions\"") +#define BENCHMARK_RESTORE_COUNTER_WARNING _Pragma("GCC diagnostic pop") +#else +#define BENCHMARK_DISABLE_COUNTER_WARNING +#define BENCHMARK_RESTORE_COUNTER_WARNING +#endif + +BENCHMARK_DISABLE_COUNTER_WARNING +#if defined(__COUNTER__) && (__COUNTER__ + 1 == __COUNTER__ + 0) +#define BENCHMARK_PRIVATE_UNIQUE_ID __COUNTER__ +#else +#define BENCHMARK_PRIVATE_UNIQUE_ID __LINE__ +#endif +BENCHMARK_RESTORE_COUNTER_WARNING + +#define BENCHMARK_PRIVATE_NAME(...) \ + BENCHMARK_PRIVATE_CONCAT(benchmark_uniq_, BENCHMARK_PRIVATE_UNIQUE_ID, \ + __VA_ARGS__) + +#define BENCHMARK_PRIVATE_CONCAT(a, b, c) BENCHMARK_PRIVATE_CONCAT2(a, b, c) +#define BENCHMARK_PRIVATE_CONCAT2(a, b, c) a##b##c +#define BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method) \ + BaseClass##_##Method##_Benchmark + +#define BENCHMARK_PRIVATE_DECLARE(n) \ + BENCHMARK_DISABLE_COUNTER_WARNING \ + static ::benchmark::Benchmark const* const BENCHMARK_PRIVATE_NAME(n) \ + BENCHMARK_RESTORE_COUNTER_WARNING BENCHMARK_UNUSED + +#define BENCHMARK(...) \ + BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + ::benchmark::internal::make_unique< \ + ::benchmark::internal::FunctionBenchmark>( \ + #__VA_ARGS__, \ + static_cast<::benchmark::internal::Function*>(__VA_ARGS__)))) + +#define BENCHMARK_WITH_ARG(n, a) BENCHMARK(n)->Arg((a)) +#define BENCHMARK_WITH_ARG2(n, a1, a2) BENCHMARK(n)->Args({(a1), (a2)}) +#define BENCHMARK_WITH_UNIT(n, t) BENCHMARK(n)->Unit((t)) +#define BENCHMARK_RANGE(n, lo, hi) BENCHMARK(n)->Range((lo), (hi)) +#define BENCHMARK_RANGE2(n, l1, h1, l2, h2) \ + BENCHMARK(n)->RangePair({{(l1), (h1)}, {(l2), (h2)}}) + +#define BENCHMARK_CAPTURE(func, test_case_name, ...) \ + BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + ::benchmark::internal::make_unique< \ + ::benchmark::internal::FunctionBenchmark>( \ + #func "/" #test_case_name, \ + [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) + +#define BENCHMARK_NAMED(func, test_case_name) \ + BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + ::benchmark::internal::make_unique< \ + ::benchmark::internal::FunctionBenchmark>( \ + #func "/" #test_case_name, \ + static_cast<::benchmark::internal::Function*>(func)))) + +#define BENCHMARK_TEMPLATE1(n, a) \ + BENCHMARK_PRIVATE_DECLARE(n) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + ::benchmark::internal::make_unique< \ + ::benchmark::internal::FunctionBenchmark>( \ + #n "<" #a ">", \ + static_cast<::benchmark::internal::Function*>(n)))) + +#define BENCHMARK_TEMPLATE2(n, a, b) \ + BENCHMARK_PRIVATE_DECLARE(n) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + ::benchmark::internal::make_unique< \ + ::benchmark::internal::FunctionBenchmark>( \ + #n "<" #a "," #b ">", \ + static_cast<::benchmark::internal::Function*>(n)))) + +#define BENCHMARK_TEMPLATE(n, ...) \ + BENCHMARK_PRIVATE_DECLARE(n) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + ::benchmark::internal::make_unique< \ + ::benchmark::internal::FunctionBenchmark>( \ + #n "<" #__VA_ARGS__ ">", \ + static_cast<::benchmark::internal::Function*>(n<__VA_ARGS__>)))) + +#define BENCHMARK_TEMPLATE1_CAPTURE(func, a, test_case_name, ...) \ + BENCHMARK_CAPTURE(func, test_case_name, __VA_ARGS__) + +#define BENCHMARK_TEMPLATE2_CAPTURE(func, a, b, test_case_name, ...) \ + BENCHMARK_PRIVATE_DECLARE(func) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + ::benchmark::internal::make_unique< \ + ::benchmark::internal::FunctionBenchmark>( \ + #func "<" #a "," #b ">" \ + "/" #test_case_name, \ + [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) + +#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "/" #Method); \ + } \ + \ + protected: \ + void BenchmarkCase(::benchmark::State&) override; \ + }; + +#define BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "<" #a ">/" #Method); \ + } \ + \ + protected: \ + void BenchmarkCase(::benchmark::State&) override; \ + }; + +#define BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "<" #a "," #b ">/" #Method); \ + } \ + \ + protected: \ + void BenchmarkCase(::benchmark::State&) override; \ + }; + +#define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, ...) \ + class BaseClass##_##Method##_Benchmark : public BaseClass<__VA_ARGS__> { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "<" #__VA_ARGS__ ">/" #Method); \ + } \ + \ + protected: \ + void BenchmarkCase(::benchmark::State&) override; \ + }; + +#define BENCHMARK_DEFINE_F(BaseClass, Method) \ + BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase + +#define BENCHMARK_TEMPLATE1_DEFINE_F(BaseClass, Method, a) \ + BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase + +#define BENCHMARK_TEMPLATE2_DEFINE_F(BaseClass, Method, a, b) \ + BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase + +#define BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, ...) \ + BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, __VA_ARGS__) \ + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase + +#define BENCHMARK_REGISTER_F(BaseClass, Method) \ + BENCHMARK_PRIVATE_REGISTER_F(BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)) + +#define BENCHMARK_PRIVATE_REGISTER_F(TestName) \ + BENCHMARK_PRIVATE_DECLARE(TestName) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + ::benchmark::internal::make_unique())) + +#define BENCHMARK_TEMPLATE_PRIVATE_CONCAT_NAME_F(BaseClass, Method) \ + BaseClass##_##Method##_BenchmarkTemplate + +#define BENCHMARK_TEMPLATE_METHOD_F(BaseClass, Method) \ + template \ + class BENCHMARK_TEMPLATE_PRIVATE_CONCAT_NAME_F(BaseClass, Method) \ + : public BaseClass { \ + protected: \ + using Base = BaseClass; \ + void BenchmarkCase(::benchmark::State&) override; \ + }; \ + template \ + void BENCHMARK_TEMPLATE_PRIVATE_CONCAT_NAME_F( \ + BaseClass, Method)::BenchmarkCase + +#define BENCHMARK_TEMPLATE_PRIVATE_INSTANTIATE_F(BaseClass, Method, \ + UniqueName, ...) \ + class UniqueName : public BENCHMARK_TEMPLATE_PRIVATE_CONCAT_NAME_F( \ + BaseClass, Method)<__VA_ARGS__> { \ + public: \ + UniqueName() { this->SetName(#BaseClass "<" #__VA_ARGS__ ">/" #Method); } \ + }; \ + BENCHMARK_PRIVATE_DECLARE(BaseClass##_##Method##_Benchmark) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + ::benchmark::internal::make_unique())) + +#define BENCHMARK_TEMPLATE_INSTANTIATE_F(BaseClass, Method, ...) \ + BENCHMARK_DISABLE_COUNTER_WARNING \ + BENCHMARK_TEMPLATE_PRIVATE_INSTANTIATE_F( \ + BaseClass, Method, BENCHMARK_PRIVATE_NAME(BaseClass##Method), \ + __VA_ARGS__) \ + BENCHMARK_RESTORE_COUNTER_WARNING + +#define BENCHMARK_F(BaseClass, Method) \ + BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ + BENCHMARK_REGISTER_F(BaseClass, Method); \ + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase + +#define BENCHMARK_TEMPLATE1_F(BaseClass, Method, a) \ + BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ + BENCHMARK_REGISTER_F(BaseClass, Method); \ + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase + +#define BENCHMARK_TEMPLATE2_F(BaseClass, Method, a, b) \ + BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ + BENCHMARK_REGISTER_F(BaseClass, Method); \ + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase + +#define BENCHMARK_TEMPLATE_F(BaseClass, Method, ...) \ + BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, __VA_ARGS__) \ + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase + +#define BENCHMARK_MAIN() \ + int main(int argc, char** argv) { \ + benchmark::MaybeReenterWithoutASLR(argc, argv); \ + char arg0_default[] = "benchmark"; \ + char* args_default = reinterpret_cast(arg0_default); \ + if (!argv) { \ + argc = 1; \ + argv = &args_default; \ + } \ + ::benchmark::Initialize(&argc, argv); \ + if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1; \ + ::benchmark::RunSpecifiedBenchmarks(); \ + ::benchmark::Shutdown(); \ + return 0; \ + } \ + int main(int, char**) + +#endif // BENCHMARK_REGISTRATION_H_ diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h new file mode 100644 index 0000000000..3faa21d385 --- /dev/null +++ b/include/benchmark/reporter.h @@ -0,0 +1,251 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BENCHMARK_REPORTER_H_ +#define BENCHMARK_REPORTER_H_ + +#if defined(_MSC_VER) +#pragma warning(push) +// C4251: needs to have dll-interface to be used by clients of class +#pragma warning(disable : 4251) +#endif + +#include +#include +#include +#include +#include + +#include "benchmark/counter.h" +#include "benchmark/macros.h" +#include "benchmark/managers.h" +#include "benchmark/statistics.h" +#include "benchmark/sysinfo.h" +#include "benchmark/types.h" + +namespace benchmark { + +namespace internal { +class BenchmarkInstance; +} // namespace internal + +struct BENCHMARK_EXPORT BenchmarkName { + std::string function_name; + std::string args; + std::string min_time; + std::string min_warmup_time; + std::string iterations; + std::string repetitions; + std::string time_type; + std::string threads; + + std::string str() const; +}; + +class BENCHMARK_EXPORT BenchmarkReporter { + public: + struct Context { + CPUInfo const& cpu_info; + SystemInfo const& sys_info; + size_t name_field_width = 0; + static const char* executable_name; + Context(); + }; + + struct BENCHMARK_EXPORT Run { + static const int64_t no_repetition_index = -1; + enum RunType { RT_Iteration, RT_Aggregate }; + + Run() + : run_type(RT_Iteration), + aggregate_unit(kTime), + skipped(internal::NotSkipped), + iterations(1), + threads(1), + time_unit(kNanosecond), + real_accumulated_time(0), + cpu_accumulated_time(0), + max_heapbytes_used(0), + use_real_time_for_initial_big_o(false), + complexity(oNone), + complexity_lambda(), + complexity_n(0), + statistics(), + report_big_o(false), + report_rms(false), + allocs_per_iter(0.0) {} + + std::string benchmark_name() const; + BenchmarkName run_name; + int64_t family_index; + int64_t per_family_instance_index; + RunType run_type; + std::string aggregate_name; + StatisticUnit aggregate_unit; + std::string report_label; + internal::Skipped skipped; + std::string skip_message; + + IterationCount iterations; + int64_t threads; + int64_t repetition_index; + int64_t repetitions; + TimeUnit time_unit; + double real_accumulated_time; + double cpu_accumulated_time; + + double GetAdjustedRealTime() const; + double GetAdjustedCPUTime() const; + + double max_heapbytes_used; + bool use_real_time_for_initial_big_o; + BigO complexity; + BigOFunc* complexity_lambda; + ComplexityN complexity_n; + const std::vector* statistics; + bool report_big_o; + bool report_rms; + UserCounters counters; + MemoryManager::Result memory_result; + double allocs_per_iter; + }; + + struct PerFamilyRunReports { + PerFamilyRunReports() : num_runs_total(0), num_runs_done(0) {} + int num_runs_total; + int num_runs_done; + std::vector Runs; + }; + + BenchmarkReporter(); + virtual bool ReportContext(const Context& context) = 0; + virtual void ReportRunsConfig(double /*min_time*/, + bool /*has_explicit_iters*/, + IterationCount /*iters*/) {} + virtual void ReportRuns(const std::vector& report) = 0; + virtual void Finalize() {} + + // Called instead of running the benchmarks when `--benchmark_list_tests` + // is specified, with the benchmarks that were selected to run. The default + // implementation prints one benchmark name per line without any markup. + virtual void List(const std::vector& benchmarks); + + void SetOutputStream(std::ostream* out) { + assert(out); + output_stream_ = out; + } + void SetErrorStream(std::ostream* err) { + assert(err); + error_stream_ = err; + } + std::ostream& GetOutputStream() const { return *output_stream_; } + std::ostream& GetErrorStream() const { return *error_stream_; } + virtual ~BenchmarkReporter(); + static void PrintBasicContext(std::ostream* out, Context const& context); + + private: + std::ostream* output_stream_; + std::ostream* error_stream_; +}; + +class BENCHMARK_EXPORT ConsoleReporter : public BenchmarkReporter { + public: + enum OutputOptions { + OO_None = 0, + OO_Color = 1, + OO_Tabular = 2, + OO_ColorTabular = OO_Color | OO_Tabular, + OO_Defaults = OO_ColorTabular + }; + explicit ConsoleReporter(OutputOptions opts_ = OO_Defaults) + : output_options_(opts_), name_field_width_(0), printed_header_(false) {} + + bool ReportContext(const Context& context) override; + void ReportRuns(const std::vector& reports) override; + + protected: + virtual void PrintRunData(const Run& result); + virtual void PrintHeader(const Run& run); + + OutputOptions output_options_; + size_t name_field_width_; + UserCounters prev_counters_; + bool printed_header_; +}; + +class BENCHMARK_EXPORT JSONReporter : public BenchmarkReporter { + public: + JSONReporter() : first_report_(true) {} + bool ReportContext(const Context& context) override; + void ReportRuns(const std::vector& reports) override; + void Finalize() override; + void List( + const std::vector& benchmarks) override; + + private: + void PrintRunData(const Run& run); + bool first_report_; +}; + +class BENCHMARK_EXPORT BENCHMARK_DEPRECATED_MSG( + "The CSV Reporter will be removed in a future release") CSVReporter + : public BenchmarkReporter { + public: + CSVReporter() : printed_header_(false) {} + bool ReportContext(const Context& context) override; + void ReportRuns(const std::vector& reports) override; + void List( + const std::vector& benchmarks) override; + + private: + void PrintRunData(const Run& run); + bool printed_header_; + std::set user_counter_names_; +}; + +inline const char* GetTimeUnitString(TimeUnit unit) { + switch (unit) { + case kSecond: + return "s"; + case kMillisecond: + return "ms"; + case kMicrosecond: + return "us"; + case kNanosecond: + return "ns"; + } + BENCHMARK_UNREACHABLE(); +} + +inline double GetTimeUnitMultiplier(TimeUnit unit) { + switch (unit) { + case kSecond: + return 1; + case kMillisecond: + return 1e3; + case kMicrosecond: + return 1e6; + case kNanosecond: + return 1e9; + } + BENCHMARK_UNREACHABLE(); +} + +} // namespace benchmark + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +#endif // BENCHMARK_REPORTER_H_ diff --git a/include/benchmark/state.h b/include/benchmark/state.h new file mode 100644 index 0000000000..356c5509a0 --- /dev/null +++ b/include/benchmark/state.h @@ -0,0 +1,282 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BENCHMARK_STATE_H_ +#define BENCHMARK_STATE_H_ + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4251 4324) +#endif + +#include +#include +#include + +#include "benchmark/counter.h" +#include "benchmark/macros.h" +#include "benchmark/statistics.h" +#include "benchmark/types.h" + +namespace benchmark { + +namespace internal { +class BenchmarkInstance; +class ThreadTimer; +class ThreadManager; +class PerfCountersMeasurement; +} // namespace internal + +class ProfilerManager; + +class BENCHMARK_EXPORT BENCHMARK_INTERNAL_CACHELINE_ALIGNED State { + public: + struct StateIterator; + friend struct StateIterator; + + inline BENCHMARK_ALWAYS_INLINE StateIterator begin(); + inline BENCHMARK_ALWAYS_INLINE StateIterator end(); + + inline bool KeepRunning(); + + inline bool KeepRunningBatch(IterationCount n); + + void PauseTiming(); + + void ResumeTiming(); + + void SkipWithMessage(const std::string& msg); + + void SkipWithError(const std::string& msg); + + bool skipped() const { return internal::NotSkipped != skipped_; } + + bool error_occurred() const { return internal::SkippedWithError == skipped_; } + + void SetIterationTime(double seconds); + + BENCHMARK_ALWAYS_INLINE + void SetBytesProcessed(int64_t bytes) { + counters["bytes_per_second"] = + Counter(static_cast(bytes), Counter::kIsRate, Counter::kIs1024); + } + + BENCHMARK_ALWAYS_INLINE + int64_t bytes_processed() const { + if (counters.find("bytes_per_second") != counters.end()) + return static_cast(counters.at("bytes_per_second")); + return 0; + } + + BENCHMARK_ALWAYS_INLINE + void SetComplexityN(ComplexityN complexity_n) { + complexity_n_ = complexity_n; + } + + BENCHMARK_ALWAYS_INLINE + ComplexityN complexity_length_n() const { return complexity_n_; } + + BENCHMARK_ALWAYS_INLINE + void SetItemsProcessed(int64_t items) { + counters["items_per_second"] = + Counter(static_cast(items), benchmark::Counter::kIsRate); + } + + BENCHMARK_ALWAYS_INLINE + int64_t items_processed() const { + if (counters.find("items_per_second") != counters.end()) + return static_cast(counters.at("items_per_second")); + return 0; + } + + void SetLabel(const std::string& label); + + BENCHMARK_ALWAYS_INLINE + int64_t range(std::size_t pos = 0) const { + assert(range_.size() > pos); + return range_[pos]; + } + + BENCHMARK_DEPRECATED_MSG("use 'range(0)' instead") + int64_t range_x() const { return range(0); } + + BENCHMARK_DEPRECATED_MSG("use 'range(1)' instead") + int64_t range_y() const { return range(1); } + + BENCHMARK_ALWAYS_INLINE + int threads() const { return threads_; } + + BENCHMARK_ALWAYS_INLINE + int thread_index() const { return thread_index_; } + + BENCHMARK_ALWAYS_INLINE + IterationCount iterations() const { + if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) { + return 0; + } + return max_iterations - total_iterations_ + batch_leftover_; + } + + BENCHMARK_ALWAYS_INLINE + std::string name() const { return name_; } + + size_t range_size() const { return range_.size(); } + + private: + IterationCount total_iterations_; + + IterationCount batch_leftover_; + + public: + const IterationCount max_iterations; + + private: + bool started_; + bool finished_; + internal::Skipped skipped_; + + std::vector range_; + + ComplexityN complexity_n_; + + public: + UserCounters counters; + + private: + State(std::string name, IterationCount max_iters, + const std::vector& ranges, int thread_i, int n_threads, + internal::ThreadTimer* timer, internal::ThreadManager* manager, + internal::PerfCountersMeasurement* perf_counters_measurement, + ProfilerManager* profiler_manager); + + void StartKeepRunning(); + inline bool KeepRunningInternal(IterationCount n, bool is_batch); + void FinishKeepRunning(); + + const std::string name_; + const int thread_index_; + const int threads_; + + internal::ThreadTimer* const timer_; + internal::ThreadManager* const manager_; + internal::PerfCountersMeasurement* const perf_counters_measurement_; + ProfilerManager* const profiler_manager_; + + friend class internal::BenchmarkInstance; +}; + +inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunning() { + return KeepRunningInternal(1, /*is_batch=*/false); +} + +inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningBatch(IterationCount n) { + return KeepRunningInternal(n, /*is_batch=*/true); +} + +inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(IterationCount n, + bool is_batch) { + assert(n > 0); + assert(is_batch || n == 1); + if (BENCHMARK_BUILTIN_EXPECT(total_iterations_ >= n, true)) { + total_iterations_ -= n; + return true; + } + if (!started_) { + StartKeepRunning(); + if (!skipped() && total_iterations_ >= n) { + total_iterations_ -= n; + return true; + } + } + if (is_batch && total_iterations_ != 0) { + batch_leftover_ = n - total_iterations_; + total_iterations_ = 0; + return true; + } + FinishKeepRunning(); + return false; +} + +struct State::StateIterator { + struct BENCHMARK_UNUSED Value {}; + typedef std::forward_iterator_tag iterator_category; + typedef Value value_type; + typedef Value reference; + typedef Value pointer; + typedef std::ptrdiff_t difference_type; + + private: + friend class State; + BENCHMARK_ALWAYS_INLINE + StateIterator() : cached_(0), parent_() {} + + BENCHMARK_ALWAYS_INLINE + explicit StateIterator(State* st) + : cached_(st->skipped() ? 0 : st->max_iterations), parent_(st) {} + + public: + BENCHMARK_ALWAYS_INLINE + Value operator*() const { return Value(); } + + BENCHMARK_ALWAYS_INLINE + StateIterator& operator++() { + assert(cached_ > 0); + --cached_; + return *this; + } + + BENCHMARK_ALWAYS_INLINE + bool operator!=(StateIterator const&) const { + if (BENCHMARK_BUILTIN_EXPECT(cached_ != 0, true)) return true; + parent_->FinishKeepRunning(); + return false; + } + + private: + IterationCount cached_; + State* const parent_; +}; + +inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::begin() { + return StateIterator(this); +} +inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::end() { + StartKeepRunning(); + return StateIterator(); +} + +class ScopedPauseTiming { + public: + explicit ScopedPauseTiming(State& state) : state_(state) { + state_.PauseTiming(); + } + ~ScopedPauseTiming() { state_.ResumeTiming(); } + + ScopedPauseTiming(const ScopedPauseTiming&) = delete; + void operator=(const ScopedPauseTiming&) = delete; + + ScopedPauseTiming(ScopedPauseTiming&&) = delete; + void operator=(ScopedPauseTiming&&) = delete; + + private: + State& state_; +}; + +} // namespace benchmark + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +#endif // BENCHMARK_STATE_H_ diff --git a/include/benchmark/statistics.h b/include/benchmark/statistics.h new file mode 100644 index 0000000000..04f01b151c --- /dev/null +++ b/include/benchmark/statistics.h @@ -0,0 +1,65 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BENCHMARK_STATISTICS_H_ +#define BENCHMARK_STATISTICS_H_ + +#include +#include + +#include "benchmark/types.h" + +namespace benchmark { + +enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda }; + +typedef int64_t ComplexityN; + +enum StatisticUnit { kTime, kPercentage }; + +typedef double(BigOFunc)(ComplexityN); + +typedef double(StatisticsFunc)(const std::vector&); + +namespace internal { +struct Statistics { + std::string name_; + StatisticsFunc* compute_; + StatisticUnit unit_; + + Statistics(const std::string& name, StatisticsFunc* compute, + StatisticUnit unit = kTime) + : name_(name), compute_(compute), unit_(unit) {} +}; + +enum AggregationReportMode : unsigned { + ARM_Unspecified = 0, + ARM_Default = 1U << 0U, + ARM_FileReportAggregatesOnly = 1U << 1U, + ARM_DisplayReportAggregatesOnly = 1U << 2U, + ARM_ReportAggregatesOnly = + ARM_FileReportAggregatesOnly | ARM_DisplayReportAggregatesOnly +}; + +enum Skipped : unsigned { + NotSkipped = 0, + SkippedWithMessage, + SkippedWithError +}; + +} // namespace internal + +} // namespace benchmark + +#endif // BENCHMARK_STATISTICS_H_ diff --git a/include/benchmark/sysinfo.h b/include/benchmark/sysinfo.h new file mode 100644 index 0000000000..711e64d7fb --- /dev/null +++ b/include/benchmark/sysinfo.h @@ -0,0 +1,71 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BENCHMARK_SYSINFO_H_ +#define BENCHMARK_SYSINFO_H_ + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4251) +#endif + +#include +#include + +#include "benchmark/macros.h" + +namespace benchmark { + +struct BENCHMARK_EXPORT CPUInfo { + struct CacheInfo { + std::string type; + int level; + int size; + int num_sharing; + }; + + enum Scaling { UNKNOWN, ENABLED, DISABLED }; + + int num_cpus; + Scaling scaling; + double cycles_per_second; + std::vector caches; + std::vector load_avg; + + static const CPUInfo& Get(); + + private: + CPUInfo(); + BENCHMARK_DISALLOW_COPY_AND_ASSIGN(CPUInfo); +}; + +struct BENCHMARK_EXPORT SystemInfo { + enum class ASLR { UNKNOWN, ENABLED, DISABLED }; + + std::string name; + ASLR ASLRStatus; + static const SystemInfo& Get(); + + private: + SystemInfo(); + BENCHMARK_DISALLOW_COPY_AND_ASSIGN(SystemInfo); +}; + +} // namespace benchmark + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +#endif // BENCHMARK_SYSINFO_H_ diff --git a/include/benchmark/types.h b/include/benchmark/types.h new file mode 100644 index 0000000000..a82ffb90cc --- /dev/null +++ b/include/benchmark/types.h @@ -0,0 +1,50 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BENCHMARK_TYPES_H_ +#define BENCHMARK_TYPES_H_ + +#include + +#include +#include +#include + +#include "benchmark/export.h" + +namespace benchmark { + +namespace internal { +#if (__cplusplus < 201402L || (defined(_MSC_VER) && _MSVC_LANG < 201402L)) +template +std::unique_ptr make_unique(Args&&... args) { + return std::unique_ptr(new T(std::forward(args)...)); +} +#else +using ::std::make_unique; +#endif +} // namespace internal + +class BenchmarkReporter; +class State; + +using IterationCount = int64_t; + +using callback_function = std::function; + +enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond, kSecond }; + +} // namespace benchmark + +#endif // BENCHMARK_TYPES_H_ diff --git a/include/benchmark/utils.h b/include/benchmark/utils.h new file mode 100644 index 0000000000..dccd3b218d --- /dev/null +++ b/include/benchmark/utils.h @@ -0,0 +1,152 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BENCHMARK_UTILS_H_ +#define BENCHMARK_UTILS_H_ + +#include +#include +#include + +#include "benchmark/export.h" +#include "benchmark/macros.h" + +namespace benchmark { + +namespace internal { +BENCHMARK_EXPORT void UseCharPointer(char const volatile*); +} + +#if (!defined(__GNUC__) && !defined(__clang__)) || defined(__pnacl__) || \ + defined(__EMSCRIPTEN__) +#define BENCHMARK_HAS_NO_INLINE_ASSEMBLY +#endif + +inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { + std::atomic_signal_fence(std::memory_order_acq_rel); +} + +#define BENCHMARK_DONOTOPTIMIZE_CONST_REF_DEPRECATED_MSG \ + "DoNotOptimize(T const&) can permit undesired compiler optimizations. " \ + "Pass a non-const lvalue instead; if the argument is an expression result, " \ + "store it in a local variable first." + +#ifndef BENCHMARK_HAS_NO_INLINE_ASSEMBLY +#if !defined(__GNUC__) || defined(__llvm__) || defined(__INTEL_COMPILER) +template +BENCHMARK_DEPRECATED_MSG(BENCHMARK_DONOTOPTIMIZE_CONST_REF_DEPRECATED_MSG) +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { + asm volatile("" : : "r,m"(value) : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { +#if defined(__clang__) + asm volatile("" : "+r,m"(value) : : "memory"); +#else + asm volatile("" : "+m,r"(value) : : "memory"); +#endif +} + +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp&& value) { +#if defined(__clang__) + asm volatile("" : "+r,m"(value) : : "memory"); +#else + asm volatile("" : "+m,r"(value) : : "memory"); +#endif +} +#elif (__GNUC__ >= 5) +template +BENCHMARK_DEPRECATED_MSG(BENCHMARK_DONOTOPTIMIZE_CONST_REF_DEPRECATED_MSG) +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value && + (sizeof(Tp) <= sizeof(Tp*))>::type + DoNotOptimize(Tp const& value) { + asm volatile("" : : "r,m"(value) : "memory"); +} + +template +BENCHMARK_DEPRECATED_MSG(BENCHMARK_DONOTOPTIMIZE_CONST_REF_DEPRECATED_MSG) +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value || + (sizeof(Tp) > sizeof(Tp*))>::type + DoNotOptimize(Tp const& value) { + asm volatile("" : : "m"(value) : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value && + (sizeof(Tp) <= sizeof(Tp*))>::type + DoNotOptimize(Tp& value) { + asm volatile("" : "+m,r"(value) : : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value || + (sizeof(Tp) > sizeof(Tp*))>::type + DoNotOptimize(Tp& value) { + asm volatile("" : "+m"(value) : : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value && + (sizeof(Tp) <= sizeof(Tp*))>::type + DoNotOptimize(Tp&& value) { + asm volatile("" : "+m,r"(value) : : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value || + (sizeof(Tp) > sizeof(Tp*))>::type + DoNotOptimize(Tp&& value) { + asm volatile("" : "+m"(value) : : "memory"); +} +#endif + +#elif defined(_MSC_VER) +template +BENCHMARK_DEPRECATED_MSG(BENCHMARK_DONOTOPTIMIZE_CONST_REF_DEPRECATED_MSG) +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { + internal::UseCharPointer(&reinterpret_cast(value)); + ClobberMemory(); +} + +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { + internal::UseCharPointer(&reinterpret_cast(value)); + ClobberMemory(); +} + +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp&& value) { + internal::UseCharPointer(&reinterpret_cast(value)); + ClobberMemory(); +} +#else +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp&& value) { + internal::UseCharPointer(&reinterpret_cast(value)); +} +#endif + +#undef BENCHMARK_DONOTOPTIMIZE_CONST_REF_DEPRECATED_MSG + +} // end namespace benchmark + +#endif // BENCHMARK_UTILS_H_ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..24b4f7407b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,76 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "google_benchmark" +description = "A library to benchmark code snippets." +requires-python = ">=3.10" +license = "Apache-2.0" +keywords = ["benchmark"] + +authors = [{ name = "Google", email = "benchmark-discuss@googlegroups.com" }] + +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Testing", + "Topic :: System :: Benchmark", +] + +dynamic = ["readme", "version"] + +[dependency-groups] +dev = ["pre-commit>=3.3.3"] + +[project.urls] +Homepage = "https://github.com/google/benchmark" +Documentation = "https://github.com/google/benchmark/tree/main/docs" +Repository = "https://github.com/google/benchmark.git" +Discord = "https://discord.gg/cz7UX7wKC2" + +[tool.setuptools] +package-dir = { "" = "bindings/python" } +zip-safe = false + +[tool.setuptools.packages.find] +where = ["bindings/python"] + +[tool.setuptools.dynamic] +readme = { file = "README.md", content-type = "text/markdown" } +version = { attr = "google_benchmark.__version__" } + +[tool.mypy] +check_untyped_defs = true +disallow_incomplete_defs = true +pretty = true +python_version = "3.11" +strict_optional = false +warn_unreachable = true + +[[tool.mypy.overrides]] +module = ["yaml"] +ignore_missing_imports = true + +[tool.ruff] +# explicitly tell ruff the source directory to correctly identify first-party package. +src = ["bindings/python"] + +line-length = 80 +target-version = "py311" + +[tool.ruff.lint] +# Enable pycodestyle (`E`, `W`), Pyflakes (`F`), and isort (`I`) codes by default. +select = ["ASYNC", "B", "C4", "C90", "E", "F", "I", "PERF", "PIE", "PT018", "RUF", "SIM", "UP", "W"] +ignore = [ + "PLW2901", # redefined-loop-name + "UP031", # printf-string-formatting +] + +[tool.ruff.lint.isort] +combine-as-imports = true diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 81ebfbfb6c..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -numpy == 1.21 -scipy == 1.5.4 -pandas == 1.1.5 diff --git a/setup.py b/setup.py index 78801b4219..d7807b4994 100644 --- a/setup.py +++ b/setup.py @@ -1,60 +1,75 @@ +import contextlib import os -import posixpath import platform import re import shutil import sys +from collections.abc import Generator +from pathlib import Path +from typing import Any -from distutils import sysconfig import setuptools from setuptools.command import build_ext +IS_WINDOWS = platform.system() == "Windows" +IS_MAC = platform.system() == "Darwin" +IS_LINUX = platform.system() == "Linux" -HERE = os.path.dirname(os.path.abspath(__file__)) +# hardcoded SABI-related options. Requires that each Python interpreter +# (hermetic or not) participating is of the same major-minor version. +py_limited_api = sys.version_info >= (3, 12) +options = {"bdist_wheel": {"py_limited_api": "cp312"}} if py_limited_api else {} -IS_WINDOWS = sys.platform.startswith("win") +def is_cibuildwheel() -> bool: + return os.getenv("CIBUILDWHEEL") is not None -with open("README.md", "r", encoding="utf-8") as fp: - long_description = fp.read() +@contextlib.contextmanager +def _maybe_patch_toolchains() -> Generator[None, None, None]: + """ + Patch rules_python toolchains to ignore root user error + when run in a Docker container on Linux in cibuildwheel. + """ - -def _get_version(): - """Parse the version string from __init__.py.""" - with open( - os.path.join(HERE, "bindings", "python", "google_benchmark", "__init__.py") - ) as init_file: - try: - version_line = next( - line for line in init_file if line.startswith("__version__") - ) - except StopIteration: - raise ValueError("__version__ not defined in __init__.py") + def fmt_toolchain_args(matchobj): + suffix = "ignore_root_user_error = True" + callargs = matchobj.group(1) + # toolchain def is broken over multiple lines + if callargs.endswith("\n"): + callargs = callargs + " " + suffix + ",\n" + # toolchain def is on one line. else: - namespace = {} - exec(version_line, namespace) # pylint: disable=exec-used - return namespace["__version__"] - - -def _parse_requirements(path): - with open(os.path.join(HERE, path)) as requirements: - return [ - line.rstrip() - for line in requirements - if not (line.isspace() or line.startswith("#")) - ] + callargs = callargs + ", " + suffix + return "python.toolchain(" + callargs + ")" + + CIBW_LINUX = is_cibuildwheel() and IS_LINUX + module_bazel = Path("MODULE.bazel") + content: str = module_bazel.read_text() + try: + if CIBW_LINUX: + module_bazel.write_text( + re.sub( + r"python.toolchain\(([\w\"\s,.=]*)\)", + fmt_toolchain_args, + content, + ) + ) + yield + finally: + if CIBW_LINUX: + module_bazel.write_text(content) class BazelExtension(setuptools.Extension): """A C/C++ extension that is defined as a Bazel BUILD target.""" - def __init__(self, name, bazel_target): + def __init__(self, name: str, bazel_target: str, **kwargs: Any): + super().__init__(name=name, sources=[], **kwargs) + self.bazel_target = bazel_target - self.relpath, self.target_name = posixpath.relpath(bazel_target, "//").split( - ":" - ) - setuptools.Extension.__init__(self, name, sources=[]) + stripped_target = bazel_target.split("//")[-1] + self.relpath, self.target_name = stripped_target.split(":") class BuildBazelExtension(build_ext.build_ext): @@ -63,96 +78,94 @@ class BuildBazelExtension(build_ext.build_ext): def run(self): for ext in self.extensions: self.bazel_build(ext) - build_ext.build_ext.run(self) + # explicitly call `bazel shutdown` for graceful exit + self.spawn(["bazel", "shutdown"]) - def bazel_build(self, ext): - """Runs the bazel build to create the package.""" - with open("WORKSPACE", "r") as workspace: - workspace_contents = workspace.read() + def copy_extensions_to_source(self): + """ + Copy generated extensions into the source tree. + This is done in the ``bazel_build`` method, so it's not necessary to + do again in the `build_ext` base class. + """ - with open("WORKSPACE", "w") as workspace: - workspace.write( - re.sub( - r'(?<=path = ").*(?=", # May be overwritten by setup\.py\.)', - sysconfig.get_python_inc().replace(os.path.sep, posixpath.sep), - workspace_contents, - ) - ) + def bazel_build(self, ext: BazelExtension) -> None: # noqa: C901 + """Runs the bazel build to create the package.""" + temp_path = Path(self.build_temp) - if not os.path.exists(self.build_temp): - os.makedirs(self.build_temp) + # We round to the minor version, which makes rules_python + # look up the latest available patch version internally. + python_version = "{}.{}".format(*sys.version_info[:2]) bazel_argv = [ "bazel", - "build", + "run", ext.bazel_target, - "--symlink_prefix=" + os.path.join(self.build_temp, "bazel-"), - "--compilation_mode=" + ("dbg" if self.debug else "opt"), + f"--symlink_prefix={temp_path / 'bazel-'}", + f"--compilation_mode={'dbg' if self.debug else 'opt'}", + # C++17 is required by nanobind + f"--cxxopt={'/std:c++17' if IS_WINDOWS else '-std=c++17'}", + f"--@rules_python//python/config_settings:python_version={python_version}", ] + if ext.py_limited_api: + bazel_argv += ["--@nanobind_bazel//:py-limited-api=cp312"] + if IS_WINDOWS: # Link with python*.lib. for library_dir in self.library_dirs: bazel_argv.append("--linkopt=/LIBPATH:" + library_dir) - elif sys.platform == "darwin" and platform.machine() == "x86_64": - bazel_argv.append("--macos_minimum_os=10.9") + elif IS_MAC: + # C++17 needs macOS 10.14 at minimum + bazel_argv.append("--macos_minimum_os=10.14") - # ARCHFLAGS is always set by cibuildwheel before macOS wheel builds. - archflags = os.getenv("ARCHFLAGS", "") - if "arm64" in archflags: - bazel_argv.append("--cpu=darwin_arm64") - bazel_argv.append("--macos_cpus=arm64") + with _maybe_patch_toolchains(): + self.spawn(bazel_argv) - self.spawn(bazel_argv) + if IS_WINDOWS: + suffix = ".pyd" + else: + suffix = ".abi3.so" if ext.py_limited_api else ".so" + + # copy the Bazel build artifacts into setuptools' libdir, + # from where the wheel is built. + pkgname = "google_benchmark" + pythonroot = Path("bindings") / "python" / "google_benchmark" + srcdir = temp_path / "bazel-bin" / pythonroot + if not self.inplace: + libdir = Path(self.build_lib) / pkgname + else: + build_py = self.get_finalized_command("build_py") + libdir = Path(build_py.get_package_dir(pkgname)) - shared_lib_suffix = '.dll' if IS_WINDOWS else '.so' - ext_bazel_bin_path = os.path.join( - self.build_temp, 'bazel-bin', - ext.relpath, ext.target_name + shared_lib_suffix) + for root, dirs, files in os.walk(srcdir, topdown=True): + # exclude runfiles directories and children. + dirs[:] = [d for d in dirs if "runfiles" not in d] - ext_dest_path = self.get_ext_fullpath(ext.name) - ext_dest_dir = os.path.dirname(ext_dest_path) - if not os.path.exists(ext_dest_dir): - os.makedirs(ext_dest_dir) - shutil.copyfile(ext_bazel_bin_path, ext_dest_path) + for f in files: + fp = Path(f) + should_copy = False + # we do not want the bare .so file included + # when building for ABI3, so we require a + # full and exact match on the file extension. + if "".join(fp.suffixes) == suffix or fp.suffix == ".pyi": + should_copy = True + elif Path(root) == srcdir and f == "py.typed": + # copy py.typed, but only at the package root. + should_copy = True - # explicitly call `bazel shutdown` for graceful exit - self.spawn(["bazel", "shutdown"]) + if should_copy: + shutil.copyfile(root / fp, libdir / fp) setuptools.setup( - name="google_benchmark", - version=_get_version(), - url="https://github.com/google/benchmark", - description="A library to benchmark code snippets.", - long_description=long_description, - long_description_content_type="text/markdown", - author="Google", - author_email="benchmark-py@google.com", - # Contained modules and scripts. - package_dir={"": "bindings/python"}, - packages=setuptools.find_packages("bindings/python"), - install_requires=_parse_requirements("bindings/python/requirements.txt"), - cmdclass=dict(build_ext=BuildBazelExtension), + cmdclass={"build_ext": BuildBazelExtension}, + package_data={"google_benchmark": ["py.typed", "*.pyi"]}, ext_modules=[ BazelExtension( - "google_benchmark._benchmark", - "//bindings/python/google_benchmark:_benchmark", + name="google_benchmark._benchmark", + bazel_target="//bindings/python/google_benchmark:benchmark_stubgen", + py_limited_api=py_limited_api, ) ], - zip_safe=False, - # PyPI package information. - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Topic :: Software Development :: Testing", - "Topic :: System :: Benchmark", - ], - license="Apache 2.0", - keywords="benchmark", + options=options, ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 70813129b2..59d3aeac0f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -# Allow the source files to find headers in src/ +#Allow the source files to find headers in src / include(GNUInstallDirs) include_directories(${PROJECT_SOURCE_DIR}/src) @@ -22,24 +22,32 @@ add_library(benchmark::benchmark ALIAS benchmark) set_target_properties(benchmark PROPERTIES OUTPUT_NAME "benchmark" VERSION ${GENERIC_LIB_VERSION} - SOVERSION 2 + SOVERSION ${GENERIC_LIB_SOVERSION} ) target_include_directories(benchmark PUBLIC $ - $ ) -generate_export_header(benchmark - EXPORT_FILE_NAME ${PROJECT_BINARY_DIR}/include/benchmark/export.h) +set_property( + SOURCE benchmark.cc + APPEND + PROPERTY COMPILE_DEFINITIONS + BENCHMARK_VERSION="${VERSION}" +) # libpfm, if available -if (HAVE_LIBPFM) - target_link_libraries(benchmark PRIVATE pfm) - add_definitions(-DHAVE_LIBPFM) +if (PFM_FOUND) + target_link_libraries(benchmark PRIVATE PFM::libpfm) + target_compile_definitions(benchmark PRIVATE -DHAVE_LIBPFM) + install( + FILES "${PROJECT_SOURCE_DIR}/cmake/Modules/FindPFM.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") endif() -# Link threads. -target_link_libraries(benchmark PRIVATE Threads::Threads) +# pthread affinity, if available +if(HAVE_PTHREAD_AFFINITY) + target_compile_definitions(benchmark PRIVATE -DBENCHMARK_HAS_PTHREAD_AFFINITY) +endif() target_link_libraries(benchmark PRIVATE ${BENCHMARK_CXX_LIBRARIES}) @@ -58,23 +66,27 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS") target_link_libraries(benchmark PRIVATE kstat) endif() +if (NOT BUILD_SHARED_LIBS) + target_compile_definitions(benchmark PUBLIC -DBENCHMARK_STATIC_DEFINE) +endif() + # Benchmark main library add_library(benchmark_main "benchmark_main.cc") add_library(benchmark::benchmark_main ALIAS benchmark_main) set_target_properties(benchmark_main PROPERTIES OUTPUT_NAME "benchmark_main" VERSION ${GENERIC_LIB_VERSION} - SOVERSION 2 + SOVERSION ${GENERIC_LIB_SOVERSION} DEFINE_SYMBOL benchmark_EXPORTS ) target_link_libraries(benchmark_main PUBLIC benchmark::benchmark) - set(generated_dir "${PROJECT_BINARY_DIR}") set(version_config "${generated_dir}/${PROJECT_NAME}ConfigVersion.cmake") set(project_config "${generated_dir}/${PROJECT_NAME}Config.cmake") set(pkg_config "${generated_dir}/${PROJECT_NAME}.pc") +set(pkg_config_main "${generated_dir}/${PROJECT_NAME}_main.pc") set(targets_to_export benchmark benchmark_main) set(targets_export_name "${PROJECT_NAME}Targets") @@ -93,8 +105,49 @@ write_basic_package_version_file( "${version_config}" VERSION ${GENERIC_LIB_VERSION} COMPATIBILITY SameMajorVersion ) +# Derive private link libraries from target +if(NOT BUILD_SHARED_LIBS) + get_target_property(LINK_LIBS benchmark LINK_LIBRARIES) + if(LINK_LIBS) + set(BENCHMARK_PRIVATE_LINK_LIBRARIES "") + foreach(LIB IN LISTS LINK_LIBS) + if(NOT TARGET "${LIB}" AND LIB MATCHES "^[a-zA-Z0-9_.-]+$") + list(APPEND BENCHMARK_PRIVATE_LINK_LIBRARIES "-l${LIB}") + elseif(TARGET "${LIB}") + get_target_property(_target_type "${LIB}" TYPE) + if(_target_type MATCHES "^(UNKNOWN|STATIC|SHARED|MODULE)_LIBRARY$") + get_target_property(_imported_loc "${LIB}" IMPORTED_LOCATION) + if(_imported_loc) + get_filename_component(_imported_dir "${_imported_loc}" DIRECTORY) + get_filename_component(_imported_name "${_imported_loc}" NAME) + string(REGEX REPLACE "^lib" "" _imported_name "${_imported_name}") + string(REGEX REPLACE "\\.(so|a|dylib)([0-9.]*)$" "" _imported_name "${_imported_name}") + list(APPEND BENCHMARK_PRIVATE_LINK_LIBRARIES "-L${_imported_dir}" "-l${_imported_name}") + endif() + endif() + endif() + endforeach() + string(JOIN " " BENCHMARK_PRIVATE_LINK_LIBRARIES ${BENCHMARK_PRIVATE_LINK_LIBRARIES}) + endif() +endif() + +function(_cmake_path output_var install_dir path_prefix) + if(IS_ABSOLUTE "${install_dir}") + set("${output_var}" "${install_dir}" PARENT_SCOPE) + else() + # FIXME: Use cmake_path(APPEND) once the minimum CMake version is 3.20. + set("${output_var}" "\${${path_prefix}}/${install_dir}" PARENT_SCOPE) + endif() +endfunction() + +_cmake_path(BENCHMARK_PKG_CONFIG_LIBDIR "${CMAKE_INSTALL_LIBDIR}" "exec_prefix") + +_cmake_path(BENCHMARK_PKG_CONFIG_INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}" "prefix") + configure_file("${PROJECT_SOURCE_DIR}/cmake/benchmark.pc.in" "${pkg_config}" @ONLY) +configure_file("${PROJECT_SOURCE_DIR}/cmake/benchmark_main.pc.in" "${pkg_config_main}" @ONLY) + export ( TARGETS ${targets_to_export} NAMESPACE "${namespace}" @@ -122,7 +175,7 @@ if (BENCHMARK_ENABLE_INSTALL) DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") install( - FILES "${pkg_config}" + FILES "${pkg_config}" "${pkg_config_main}" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") install( @@ -164,3 +217,11 @@ else() DESTINATION ${CMAKE_INSTALL_DOCDIR}) endif() endif() + +set(CMAKE_INSTALL_PYTOOLSDIR "${CMAKE_INSTALL_DATADIR}/googlebenchmark/tools" CACHE PATH "") + +if (BENCHMARK_ENABLE_INSTALL AND BENCHMARK_INSTALL_TOOLS) + install( + DIRECTORY "${PROJECT_SOURCE_DIR}/tools/" + DESTINATION ${CMAKE_INSTALL_PYTOOLSDIR}) +endif() diff --git a/src/benchmark.cc b/src/benchmark.cc index 254b95ef9e..91280295e8 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -12,25 +12,39 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "benchmark/benchmark.h" - +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/reporter.h" +#include "benchmark/state.h" +#include "benchmark/types.h" #include "benchmark_api_internal.h" #include "benchmark_runner.h" #include "internal_macros.h" #ifndef BENCHMARK_OS_WINDOWS -#ifndef BENCHMARK_OS_FUCHSIA +#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) && \ + !defined(BENCHMARK_OS_WASI) #include #endif #include #include #endif +#ifdef __ANDROID_NDK__ +#include +#endif + +#ifdef BENCHMARK_OS_LINUX +#include +#include +#endif + #include #include #include #include #include +#include #include #include #include @@ -46,7 +60,6 @@ #include "commandlineflags.h" #include "complexity.h" #include "counter.h" -#include "internal_macros.h" #include "log.h" #include "mutex.h" #include "perf_counters.h" @@ -65,16 +78,25 @@ BM_DEFINE_bool(benchmark_list_tests, false); // linked into the binary are run. BM_DEFINE_string(benchmark_filter, ""); -// Minimum number of seconds we should run benchmark before results are -// considered significant. For cpu-time based tests, this is the lower bound +// Specification of how long to run the benchmark. +// +// It can be either an exact number of iterations (specified as `x`), +// or a minimum number of seconds (specified as `s`). If the latter +// format (ie., min seconds) is used, the system may run the benchmark longer +// until the results are considered significant. +// +// For backward compatibility, the `s` suffix may be omitted, in which case, +// the specified number is interpreted as the number of seconds. +// +// For cpu-time based tests, this is the lower bound // on the total cpu time used by all threads that make up the test. For // real-time based tests, this is the lower bound on the elapsed time of the // benchmark execution, regardless of number of threads. -BM_DEFINE_double(benchmark_min_time, 0.5); +BM_DEFINE_string(benchmark_min_time, kDefaultMinTimeStr); // Minimum number of seconds a benchmark should be run before results should be -// taken into account. This e.g can be neccessary for benchmarks of code which -// needs to fill some form of cache before performance is of interrest. +// taken into account. This e.g can be necessary for benchmarks of code which +// needs to fill some form of cache before performance is of interest. // Note: results gathered within this period are discarded and not used for // reported result. BM_DEFINE_double(benchmark_min_warmup_time, 0.0); @@ -83,6 +105,11 @@ BM_DEFINE_double(benchmark_min_warmup_time, 0.0); // standard deviation of the runs will be reported. BM_DEFINE_int32(benchmark_repetitions, 1); +// If enabled, forces each benchmark to execute exactly one iteration and one +// repetition, bypassing any configured +// MinTime()/MinWarmUpTime()/Iterations()/Repetitions() +BM_DEFINE_bool(benchmark_dry_run, false); + // If set, enable random interleaving of repetitions of all benchmarks. // See http://github.com/google/benchmark/issues/1051 for details. BM_DEFINE_bool(benchmark_enable_random_interleaving, false); @@ -137,34 +164,64 @@ BM_DEFINE_int32(v, 0); namespace internal { -BENCHMARK_EXPORT std::map* global_context = nullptr; +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +std::map* global_context = nullptr; -// FIXME: wouldn't LTO mess this up? -void UseCharPointer(char const volatile*) {} +BENCHMARK_EXPORT std::map*& GetGlobalContext() { + return global_context; +} + +namespace { +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +void const volatile* volatile global_force_escape_pointer; +} // namespace + +// FIXME: Verify if LTO still messes this up? +void UseCharPointer(char const volatile* const v) { + // We want to escape the pointer `v` so that the compiler can not eliminate + // computations that produced it. To do that, we escape the pointer by storing + // it into a volatile variable, since generally, volatile store, is not + // something the compiler is allowed to elide. + global_force_escape_pointer = reinterpret_cast(v); +} } // namespace internal -State::State(IterationCount max_iters, const std::vector& ranges, - int thread_i, int n_threads, internal::ThreadTimer* timer, - internal::ThreadManager* manager, - internal::PerfCountersMeasurement* perf_counters_measurement) +State::State(std::string name, IterationCount max_iters, + const std::vector& ranges, int thread_i, int n_threads, + internal::ThreadTimer* timer, internal::ThreadManager* manager, + internal::PerfCountersMeasurement* perf_counters_measurement, + ProfilerManager* profiler_manager) : total_iterations_(0), batch_leftover_(0), max_iterations(max_iters), started_(false), finished_(false), - error_occurred_(false), + skipped_(internal::NotSkipped), range_(ranges), complexity_n_(0), + name_(std::move(name)), thread_index_(thread_i), threads_(n_threads), timer_(timer), manager_(manager), - perf_counters_measurement_(perf_counters_measurement) { + perf_counters_measurement_(perf_counters_measurement), + profiler_manager_(profiler_manager) { BM_CHECK(max_iterations != 0) << "At least one iteration must be run"; BM_CHECK_LT(thread_index_, threads_) << "thread_index must be less than threads"; + // Add counters with correct flag now. If added with `counters[name]` in + // `PauseTiming`, a new `Counter` will be inserted the first time, which + // won't have the flag. Inserting them now also reduces the allocations + // during the benchmark. + if (perf_counters_measurement_ != nullptr) { + for (const std::string& counter_name : + perf_counters_measurement_->names()) { + counters[counter_name] = Counter(0.0, Counter::kAvgIterations); + } + } + // Note: The use of offsetof below is technically undefined until C++17 // because State is not a standard layout type. However, all compilers // currently provide well-defined behavior as an extension (which is @@ -174,67 +231,97 @@ State::State(IterationCount max_iters, const std::vector& ranges, #if defined(__INTEL_COMPILER) #pragma warning push #pragma warning(disable : 1875) -#elif defined(__GNUC__) +#elif defined(__GNUC__) || defined(__clang__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" +#endif +#if defined(__NVCC__) +#pragma nv_diagnostic push +#pragma nv_diag_suppress 1427 +#endif +#if defined(__NVCOMPILER) +#pragma diagnostic push +#pragma diag_suppress offset_in_non_POD_nonstandard #endif // Offset tests to ensure commonly accessed data is on the first cache line. const int cache_line_size = 64; - static_assert(offsetof(State, error_occurred_) <= - (cache_line_size - sizeof(error_occurred_)), - ""); + static_assert( + offsetof(State, skipped_) <= (cache_line_size - sizeof(skipped_)), ""); #if defined(__INTEL_COMPILER) #pragma warning pop -#elif defined(__GNUC__) +#elif defined(__GNUC__) || defined(__clang__) #pragma GCC diagnostic pop #endif +#if defined(__NVCC__) +#pragma nv_diagnostic pop +#endif +#if defined(__NVCOMPILER) +#pragma diagnostic pop +#endif } void State::PauseTiming() { // Add in time accumulated so far - BM_CHECK(started_ && !finished_ && !error_occurred_); + BM_CHECK(started_ && !finished_ && !skipped()); timer_->StopTimer(); - if (perf_counters_measurement_) { + if (perf_counters_measurement_ != nullptr) { std::vector> measurements; if (!perf_counters_measurement_->Stop(measurements)) { BM_CHECK(false) << "Perf counters read the value failed."; } for (const auto& name_and_measurement : measurements) { - auto name = name_and_measurement.first; - auto measurement = name_and_measurement.second; - BM_CHECK_EQ(std::fpclassify((double)counters[name]), FP_ZERO); - counters[name] = Counter(measurement, Counter::kAvgIterations); + const std::string& name = name_and_measurement.first; + const double measurement = name_and_measurement.second; + // Counter was inserted with `kAvgIterations` flag by the constructor. + assert(counters.find(name) != counters.end()); + counters[name].value += measurement; } } } void State::ResumeTiming() { - BM_CHECK(started_ && !finished_ && !error_occurred_); + BM_CHECK(started_ && !finished_ && !skipped()); timer_->StartTimer(); - if (perf_counters_measurement_) { + if (perf_counters_measurement_ != nullptr) { perf_counters_measurement_->Start(); } } -void State::SkipWithError(const char* msg) { - BM_CHECK(msg); - error_occurred_ = true; +void State::SkipWithMessage(const std::string& msg) { + skipped_ = internal::SkippedWithMessage; { MutexLock l(manager_->GetBenchmarkMutex()); - if (manager_->results.has_error_ == false) { - manager_->results.error_message_ = msg; - manager_->results.has_error_ = true; + if (internal::NotSkipped == manager_->results.skipped_) { + manager_->results.skip_message_ = msg; + manager_->results.skipped_ = skipped_; } } total_iterations_ = 0; - if (timer_->running()) timer_->StopTimer(); + if (timer_->running()) { + timer_->StopTimer(); + } +} + +void State::SkipWithError(const std::string& msg) { + skipped_ = internal::SkippedWithError; + { + MutexLock l(manager_->GetBenchmarkMutex()); + if (internal::NotSkipped == manager_->results.skipped_) { + manager_->results.skip_message_ = msg; + manager_->results.skipped_ = skipped_; + } + } + total_iterations_ = 0; + if (timer_->running()) { + timer_->StopTimer(); + } } void State::SetIterationTime(double seconds) { timer_->SetIterationTime(seconds); } -void State::SetLabel(const char* label) { +void State::SetLabel(const std::string& label) { MutexLock l(manager_->GetBenchmarkMutex()); manager_->results.report_label_ = label; } @@ -242,20 +329,28 @@ void State::SetLabel(const char* label) { void State::StartKeepRunning() { BM_CHECK(!started_ && !finished_); started_ = true; - total_iterations_ = error_occurred_ ? 0 : max_iterations; + total_iterations_ = skipped() ? 0 : max_iterations; + if (BENCHMARK_BUILTIN_EXPECT(profiler_manager_ != nullptr, false)) { + profiler_manager_->AfterSetupStart(); + } manager_->StartStopBarrier(); - if (!error_occurred_) ResumeTiming(); + if (!skipped()) { + ResumeTiming(); + } } void State::FinishKeepRunning() { - BM_CHECK(started_ && (!finished_ || error_occurred_)); - if (!error_occurred_) { + BM_CHECK(started_ && (!finished_ || skipped())); + if (!skipped()) { PauseTiming(); } // Total iterations has now wrapped around past 0. Fix this. total_iterations_ = 0; finished_ = true; manager_->StartStopBarrier(); + if (BENCHMARK_BUILTIN_EXPECT(profiler_manager_ != nullptr, false)) { + profiler_manager_->BeforeTeardownStop(); + } } namespace internal { @@ -264,7 +359,9 @@ namespace { // Flushes streams after invoking reporter methods that write to them. This // ensures users get timely updates even when streams are not line-buffered. void FlushStreams(BenchmarkReporter* reporter) { - if (!reporter) return; + if (reporter == nullptr) { + return; + } std::flush(reporter->GetOutputStream()); std::flush(reporter->GetErrorStream()); } @@ -277,16 +374,20 @@ void Report(BenchmarkReporter* display_reporter, assert(reporter); // If there are no aggregates, do output non-aggregates. aggregates_only &= !results.aggregates_only.empty(); - if (!aggregates_only) reporter->ReportRuns(results.non_aggregates); - if (!results.aggregates_only.empty()) + if (!aggregates_only) { + reporter->ReportRuns(results.non_aggregates); + } + if (!results.aggregates_only.empty()) { reporter->ReportRuns(results.aggregates_only); + } }; report_one(display_reporter, run_results.display_report_aggregates_only, run_results); - if (file_reporter) + if (file_reporter != nullptr) { report_one(file_reporter, run_results.file_report_aggregates_only, run_results); + } FlushStreams(display_reporter); FlushStreams(file_reporter); @@ -307,10 +408,13 @@ void RunBenchmarks(const std::vector& benchmarks, std::max(name_field_width, benchmark.name().str().size()); might_have_aggregates |= benchmark.repetitions() > 1; - for (const auto& Stat : benchmark.statistics()) + for (const auto& Stat : benchmark.statistics()) { stat_field_width = std::max(stat_field_width, Stat.name_.size()); + } + } + if (might_have_aggregates) { + name_field_width += 1 + stat_field_width; } - if (might_have_aggregates) name_field_width += 1 + stat_field_width; // Print header here BenchmarkReporter::Context context; @@ -321,27 +425,53 @@ void RunBenchmarks(const std::vector& benchmarks, per_family_reports; if (display_reporter->ReportContext(context) && - (!file_reporter || file_reporter->ReportContext(context))) { + ((file_reporter == nullptr) || file_reporter->ReportContext(context))) { FlushStreams(display_reporter); FlushStreams(file_reporter); size_t num_repetitions_total = 0; + // This perfcounters object needs to be created before the runners vector + // below so it outlasts their lifetime. + PerfCountersMeasurement perfcounters( + StrSplit(FLAGS_benchmark_perf_counters, ',')); + + // Vector of benchmarks to run std::vector runners; runners.reserve(benchmarks.size()); + + // Count the number of benchmarks with threads to warn the user in case + // performance counters are used. + int benchmarks_with_threads = 0; + + // Loop through all benchmarks for (const BenchmarkInstance& benchmark : benchmarks) { BenchmarkReporter::PerFamilyRunReports* reports_for_family = nullptr; - if (benchmark.complexity() != oNone) + if (benchmark.complexity() != oNone) { reports_for_family = &per_family_reports[benchmark.family_index()]; - - runners.emplace_back(benchmark, reports_for_family); + } + benchmarks_with_threads += static_cast(benchmark.threads() > 1); + runners.emplace_back(benchmark, &perfcounters, reports_for_family); int num_repeats_of_this_instance = runners.back().GetNumRepeats(); - num_repetitions_total += num_repeats_of_this_instance; - if (reports_for_family) + num_repetitions_total += + static_cast(num_repeats_of_this_instance); + if (reports_for_family != nullptr) { reports_for_family->num_runs_total += num_repeats_of_this_instance; + } } assert(runners.size() == benchmarks.size() && "Unexpected runner count."); + // The use of performance counters with threads would be unintuitive for + // the average user so we need to warn them about this case + if ((benchmarks_with_threads > 0) && (perfcounters.num_counters() > 0)) { + GetErrorLogInstance() + << "***WARNING*** There are " << benchmarks_with_threads + << " benchmarks with threads and " << perfcounters.num_counters() + << " performance counters were requested. Beware counters will " + "reflect the combined usage across all " + "threads.\n"; + } + std::vector repetition_indices; repetition_indices.reserve(num_repetitions_total); for (size_t runner_index = 0, num_runners = runners.size(); @@ -362,9 +492,18 @@ void RunBenchmarks(const std::vector& benchmarks, for (size_t repetition_index : repetition_indices) { internal::BenchmarkRunner& runner = runners[repetition_index]; runner.DoOneRepetition(); - if (runner.HasRepeatsRemaining()) continue; + if (runner.HasRepeatsRemaining()) { + continue; + } // FIXME: report each repetition separately, not all of them in bulk. + display_reporter->ReportRunsConfig( + runner.GetMinTime(), runner.HasExplicitIters(), runner.GetIters()); + if (file_reporter != nullptr) { + file_reporter->ReportRunsConfig( + runner.GetMinTime(), runner.HasExplicitIters(), runner.GetIters()); + } + RunResults run_results = runner.GetResults(); // Maybe calculate complexity report @@ -384,7 +523,9 @@ void RunBenchmarks(const std::vector& benchmarks, } } display_reporter->Finalize(); - if (file_reporter) file_reporter->Finalize(); + if (file_reporter != nullptr) { + file_reporter->Finalize(); + } FlushStreams(display_reporter); FlushStreams(file_reporter); } @@ -398,14 +539,16 @@ std::unique_ptr CreateReporter( typedef std::unique_ptr PtrType; if (name == "console") { return PtrType(new ConsoleReporter(output_opts)); - } else if (name == "json") { + } + if (name == "json") { return PtrType(new JSONReporter()); - } else if (name == "csv") { + } + if (name == "csv") { return PtrType(new CSVReporter()); - } else { - std::cerr << "Unexpected format: '" << name << "'\n"; - std::exit(1); } + std::cerr << "Unexpected format: '" << name << "'\n"; + std::flush(std::cerr); + std::exit(1); } BENCHMARK_RESTORE_DEPRECATED_WARNING @@ -443,7 +586,7 @@ ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) { } // end namespace internal BenchmarkReporter* CreateDefaultDisplayReporter() { - static auto default_display_reporter = + static auto* default_display_reporter = internal::CreateReporter(FLAGS_benchmark_format, internal::GetOutputOptions()) .release(); @@ -477,14 +620,15 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter, std::string spec) { - if (spec.empty() || spec == "all") + if (spec.empty() || spec == "all") { spec = "."; // Regexp that matches all benchmarks + } // Setup the reporters std::ofstream output_file; std::unique_ptr default_display_reporter; std::unique_ptr default_file_reporter; - if (!display_reporter) { + if (display_reporter == nullptr) { default_display_reporter.reset(CreateDefaultDisplayReporter()); display_reporter = default_display_reporter.get(); } @@ -492,21 +636,26 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, auto& Err = display_reporter->GetErrorStream(); std::string const& fname = FLAGS_benchmark_out; - if (fname.empty() && file_reporter) { + if (fname.empty() && (file_reporter != nullptr)) { Err << "A custom file reporter was provided but " - "--benchmark_out= was not specified." - << std::endl; + "--benchmark_out= was not specified.\n"; + Out.flush(); + Err.flush(); std::exit(1); } if (!fname.empty()) { output_file.open(fname); if (!output_file.is_open()) { - Err << "invalid file name: '" << fname << "'" << std::endl; + Err << "invalid file name: '" << fname << "'\n"; + Out.flush(); + Err.flush(); std::exit(1); } - if (!file_reporter) { + if (file_reporter == nullptr) { default_file_reporter = internal::CreateReporter( - FLAGS_benchmark_out_format, ConsoleReporter::OO_None); + FLAGS_benchmark_out_format, FLAGS_benchmark_counters_tabular + ? ConsoleReporter::OO_Tabular + : ConsoleReporter::OO_None); file_reporter = default_file_reporter.get(); } file_reporter->SetOutputStream(&output_file); @@ -514,20 +663,27 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, } std::vector benchmarks; - if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) return 0; + if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) { + Out.flush(); + Err.flush(); + return 0; + } if (benchmarks.empty()) { Err << "Failed to match any benchmarks against regex: " << spec << "\n"; + Out.flush(); + Err.flush(); return 0; } if (FLAGS_benchmark_list_tests) { - for (auto const& benchmark : benchmarks) - Out << benchmark.name().str() << "\n"; + display_reporter->List(benchmarks); } else { internal::RunBenchmarks(benchmarks, display_reporter, file_reporter); } + Out.flush(); + Err.flush(); return benchmarks.size(); } @@ -552,11 +708,20 @@ void RegisterMemoryManager(MemoryManager* manager) { internal::memory_manager = manager; } -void AddCustomContext(const std::string& key, const std::string& value) { +void RegisterProfilerManager(ProfilerManager* manager) { + // Don't allow overwriting an existing manager. + if (manager != nullptr) { + BM_CHECK_EQ(internal::profiler_manager, nullptr); + } + internal::profiler_manager = manager; +} + +void AddCustomContext(std::string key, std::string value) { if (internal::global_context == nullptr) { internal::global_context = new std::map(); } - if (!internal::global_context->emplace(key, value).second) { + if (!internal::global_context->emplace(std::move(key), std::move(value)) + .second) { std::cerr << "Failed to add custom context \"" << key << "\" as it already " << "exists with value \"" << value << "\"\n"; } @@ -566,42 +731,28 @@ namespace internal { void (*HelperPrintf)(); +namespace { void PrintUsageAndExit() { - if (HelperPrintf) { - HelperPrintf(); - } else { - fprintf(stdout, - "benchmark" - " [--benchmark_list_tests={true|false}]\n" - " [--benchmark_filter=]\n" - " [--benchmark_min_time=]\n" - " [--benchmark_min_warmup_time=]\n" - " [--benchmark_repetitions=]\n" - " [--benchmark_enable_random_interleaving={true|false}]\n" - " [--benchmark_report_aggregates_only={true|false}]\n" - " [--benchmark_display_aggregates_only={true|false}]\n" - " [--benchmark_format=]\n" - " [--benchmark_out=]\n" - " [--benchmark_out_format=]\n" - " [--benchmark_color={auto|true|false}]\n" - " [--benchmark_counters_tabular={true|false}]\n" - " [--benchmark_context==,...]\n" - " [--benchmark_time_unit={ns|us|ms|s}]\n" - " [--v=]\n"); - } - exit(0); + HelperPrintf(); + std::flush(std::cout); + std::flush(std::cerr); + std::exit(0); } void SetDefaultTimeUnitFromFlag(const std::string& time_unit_flag) { if (time_unit_flag == "s") { return SetDefaultTimeUnit(kSecond); - } else if (time_unit_flag == "ms") { + } + if (time_unit_flag == "ms") { return SetDefaultTimeUnit(kMillisecond); - } else if (time_unit_flag == "us") { + } + if (time_unit_flag == "us") { return SetDefaultTimeUnit(kMicrosecond); - } else if (time_unit_flag == "ns") { + } + if (time_unit_flag == "ns") { return SetDefaultTimeUnit(kNanosecond); - } else if (!time_unit_flag.empty()) { + } + if (!time_unit_flag.empty()) { PrintUsageAndExit(); } } @@ -609,17 +760,18 @@ void SetDefaultTimeUnitFromFlag(const std::string& time_unit_flag) { void ParseCommandLineFlags(int* argc, char** argv) { using namespace benchmark; BenchmarkReporter::Context::executable_name = - (argc && *argc > 0) ? argv[0] : "unknown"; - for (int i = 1; argc && i < *argc; ++i) { + ((argc != nullptr) && *argc > 0) ? argv[0] : "unknown"; + for (int i = 1; (argc != nullptr) && i < *argc; ++i) { if (ParseBoolFlag(argv[i], "benchmark_list_tests", &FLAGS_benchmark_list_tests) || ParseStringFlag(argv[i], "benchmark_filter", &FLAGS_benchmark_filter) || - ParseDoubleFlag(argv[i], "benchmark_min_time", + ParseStringFlag(argv[i], "benchmark_min_time", &FLAGS_benchmark_min_time) || ParseDoubleFlag(argv[i], "benchmark_min_warmup_time", &FLAGS_benchmark_min_warmup_time) || ParseInt32Flag(argv[i], "benchmark_repetitions", &FLAGS_benchmark_repetitions) || + ParseBoolFlag(argv[i], "benchmark_dry_run", &FLAGS_benchmark_dry_run) || ParseBoolFlag(argv[i], "benchmark_enable_random_interleaving", &FLAGS_benchmark_enable_random_interleaving) || ParseBoolFlag(argv[i], "benchmark_report_aggregates_only", @@ -640,7 +792,9 @@ void ParseCommandLineFlags(int* argc, char** argv) { ParseStringFlag(argv[i], "benchmark_time_unit", &FLAGS_benchmark_time_unit) || ParseInt32Flag(argv[i], "v", &FLAGS_v)) { - for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1]; + for (int j = i; j != *argc - 1; ++j) { + argv[j] = argv[j + 1]; + } --(*argc); --i; @@ -658,22 +812,207 @@ void ParseCommandLineFlags(int* argc, char** argv) { if (FLAGS_benchmark_color.empty()) { PrintUsageAndExit(); } + if (FLAGS_benchmark_dry_run) { + AddCustomContext("dry_run", "true"); + } for (const auto& kv : FLAGS_benchmark_context) { AddCustomContext(kv.first, kv.second); } } +} // end namespace + int InitializeStreams() { static std::ios_base::Init init; return 0; } +template +std::make_unsigned_t get_as_unsigned(T v) { + using UnsignedT = std::make_unsigned_t; + return static_cast(v); +} + } // end namespace internal +#ifdef BENCHMARK_OS_LINUX +static constexpr const char kTestChildArg[] = "--benchmark_aslr_test_child="; +bool ValidateNoASLRPersonalitySticks(char* argv0) { + // Verify that the personality change survives exec() by testing in a child. + // Some LSMs (e.g., AppArmor) may reset personality flags during execve(), + // even though they were successfully set in the parent process. + // This prevents infinite re-exec loops when the kernel silently resets + // the personality after each exec. + int pipefd[2]; + if (pipe(pipefd) != 0) return false; + + pid_t pid = fork(); + if (pid == -1) { + close(pipefd[0]); + close(pipefd[1]); + return false; + } + + if (pid == 0) { + // Child: prepare to exec with test argument + close(pipefd[0]); + + // Build test argument on stack (safe before exec) + char test_arg[64]; + int test_arg_len = std::snprintf(test_arg, sizeof(test_arg), "%s%d", + kTestChildArg, pipefd[1]); + BM_CHECK_LT(static_cast(test_arg_len), sizeof(test_arg)); + (void)test_arg_len; + + // Simple argv with just the executable and test argument + char* child_argv[] = {argv0, test_arg, nullptr}; + + execv(argv0, child_argv); + // If exec fails, exit + _exit(1); + } + + // Parent: wait for child to report back + close(pipefd[1]); + + char result = 0; + ssize_t nread = read(pipefd[0], &result, 1); + close(pipefd[0]); + + int status; + waitpid(pid, &status, 0); + + // Did we successfully read the result and it indicates ADDR_NO_RANDOMIZE? + return status == 0 && nread == 1 && result == 1; +} +#endif + +void MaybeReenterWithoutASLR(int argc, char** argv) { + (void)argc; + + // On e.g. Hexagon simulator, argv may be NULL. + if (!argv) return; + +#ifdef BENCHMARK_OS_LINUX + static constexpr size_t kTestChildArgLen = sizeof(kTestChildArg) - 1; + + // Check if we are a test child process that should report personality and + // exit + if (argc == 2 && + std::strncmp(argv[1], kTestChildArg, kTestChildArgLen) == 0) { + const int write_fd = std::atoi(argv[1] + kTestChildArgLen); + const auto test_personality = personality(0xffffffff); + // Write 1 if ADDR_NO_RANDOMIZE is set, 0 otherwise + char result = + ((test_personality != -1) && + (internal::get_as_unsigned(test_personality) & ADDR_NO_RANDOMIZE)) + ? 1 + : 0; + const auto nbytes = write(write_fd, &result, 1); + close(write_fd); + if (test_personality == -1 || nbytes != 1) std::exit(1); + std::exit(0); + } + + const auto curr_personality = personality(0xffffffff); + + // We should never fail to read-only query the current personality, + // but let's be cautious. + if (curr_personality == -1) return; + + // If ASLR is already disabled, we have nothing more to do. + if (internal::get_as_unsigned(curr_personality) & ADDR_NO_RANDOMIZE) return; + + // Try to change the personality to disable ASLR. + const auto proposed_personality = + internal::get_as_unsigned(curr_personality) | ADDR_NO_RANDOMIZE; + const auto prev_personality = personality(proposed_personality); + + // Have we failed to change the personality? That may happen. + if (prev_personality == -1) return; + + // Make sure the parsona has been updated with the no-ASLR flag, + // otherwise we will try to reenter infinitely. + // This seems impossible, but can happen in some docker configurations. + const auto new_personality = personality(0xffffffff); + if (new_personality == -1) return; + if ((internal::get_as_unsigned(new_personality) & ADDR_NO_RANDOMIZE) == 0) + return; + + // Additionally, ensure that the personality change would survive exec(). + if (ValidateNoASLRPersonalitySticks(argv[0])) { + execv(argv[0], argv); + } + + // Personality doesn't survive exec() boundary, or execv() failed. + // We need to try to un-change the personality to "re-enable" ASLR, + // at least so that there is a warning in the output, + // and continue as-is. + const auto restored_personality = + internal::get_as_unsigned(new_personality) & + ~internal::get_as_unsigned(ADDR_NO_RANDOMIZE); + personality(restored_personality); + // This may or may not have failed, but we're out of options here. +#else + return; +#endif +} + +std::string GetBenchmarkVersion() { +#ifdef BENCHMARK_VERSION + return {BENCHMARK_VERSION}; +#else + return {""}; +#endif +} + +void PrintDefaultHelp() { + fprintf(stdout, + "benchmark" + " [--benchmark_list_tests={true|false}]\n" + " [--benchmark_filter=]\n" + " [--benchmark_min_time=`x` OR `s` ]\n" + " [--benchmark_min_warmup_time=]\n" + " [--benchmark_repetitions=]\n" + " [--benchmark_dry_run={true|false}]\n" + " [--benchmark_enable_random_interleaving={true|false}]\n" + " [--benchmark_report_aggregates_only={true|false}]\n" + " [--benchmark_display_aggregates_only={true|false}]\n" + " [--benchmark_format=]\n" + " [--benchmark_out=]\n" + " [--benchmark_out_format=]\n" + " [--benchmark_color={auto|true|false}]\n" + " [--benchmark_counters_tabular={true|false}]\n" +#if defined HAVE_LIBPFM + " [--benchmark_perf_counters=,...]\n" +#endif + " [--benchmark_context==,...]\n" + " [--benchmark_time_unit={ns|us|ms|s}]\n" + " [--v=]\n"); +} + void Initialize(int* argc, char** argv, void (*HelperPrintf)()) { +#ifdef __ANDROID_NDK__ + // We want standalone android benchmarks to match the performance properties + // of the production environment, so match that configuration here. + + // Android 12 (API level 31) introduced zeroing of allocated memory in bionic + // as a hardening feature; however, this is not enabled for apps. + if (__builtin_available(android 31, *)) { + BM_CHECK_EQ(mallopt(M_BIONIC_ZERO_INIT, 0), 1); + } + + // The default configuration of bionic is to return pages to the OS as soon + // as they are freed. But application processes are configured to run with a + // delay before returning memory to avoid excessive faulting on repeated + // allocation and deallocation, which is common in repeated benchmark runs. + if (__builtin_available(android 27, *)) { + BM_CHECK_EQ(mallopt(M_DECAY_TIME, 1), 1); + } +#endif + internal::HelperPrintf = HelperPrintf; internal::ParseCommandLineFlags(argc, argv); internal::LogLevel() = FLAGS_v; - internal::HelperPrintf = HelperPrintf; } void Shutdown() { delete internal::global_context; } diff --git a/src/benchmark_api_internal.cc b/src/benchmark_api_internal.cc index 963fea22f3..f9c4990ddf 100644 --- a/src/benchmark_api_internal.cc +++ b/src/benchmark_api_internal.cc @@ -7,7 +7,8 @@ namespace benchmark { namespace internal { -BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, +BenchmarkInstance::BenchmarkInstance(benchmark::Benchmark* benchmark, + int family_idx, int per_family_instance_idx, const std::vector& args, int thread_count) @@ -27,7 +28,9 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, min_time_(benchmark_.min_time_), min_warmup_time_(benchmark_.min_warmup_time_), iterations_(benchmark_.iterations_), - threads_(thread_count) { + threads_(thread_count), + setup_(benchmark_.setup_), + teardown_(benchmark_.teardown_) { name_.function_name = benchmark_.name_; size_t arg_i = 0; @@ -84,33 +87,31 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, if (!benchmark_.thread_counts_.empty()) { name_.threads = StrFormat("threads:%d", threads_); } - - setup_ = benchmark_.setup_; - teardown_ = benchmark_.teardown_; } State BenchmarkInstance::Run( IterationCount iters, int thread_id, internal::ThreadTimer* timer, internal::ThreadManager* manager, - internal::PerfCountersMeasurement* perf_counters_measurement) const { - State st(iters, args_, thread_id, threads_, timer, manager, - perf_counters_measurement); + internal::PerfCountersMeasurement* perf_counters_measurement, + ProfilerManager* profiler_manager) const { + State st(name_.function_name, iters, args_, thread_id, threads_, timer, + manager, perf_counters_measurement, profiler_manager); benchmark_.Run(st); return st; } void BenchmarkInstance::Setup() const { - if (setup_) { - State st(/*iters*/ 1, args_, /*thread_id*/ 0, threads_, nullptr, nullptr, - nullptr); + if (setup_ != nullptr) { + State st(name_.function_name, /*iters*/ 1, args_, /*thread_id*/ 0, threads_, + nullptr, nullptr, nullptr, nullptr); setup_(st); } } void BenchmarkInstance::Teardown() const { - if (teardown_) { - State st(/*iters*/ 1, args_, /*thread_id*/ 0, threads_, nullptr, nullptr, - nullptr); + if (teardown_ != nullptr) { + State st(name_.function_name, /*iters*/ 1, args_, /*thread_id*/ 0, threads_, + nullptr, nullptr, nullptr, nullptr); teardown_(st); } } diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index 94f516531b..0dd950cbe5 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -8,7 +8,9 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/reporter.h" +#include "benchmark/sysinfo.h" #include "commandlineflags.h" namespace benchmark { @@ -17,9 +19,9 @@ namespace internal { // Information kept per benchmark we may want to run class BenchmarkInstance { public: - BenchmarkInstance(Benchmark* benchmark, int family_index, - int per_family_instance_index, - const std::vector& args, int threads); + BenchmarkInstance(benchmark::Benchmark* benchmark, int family_idx, + int per_family_instance_idx, + const std::vector& args, int thread_count); const BenchmarkName& name() const { return name_; } int family_index() const { return family_index_; } @@ -41,14 +43,18 @@ class BenchmarkInstance { int threads() const { return threads_; } void Setup() const; void Teardown() const; + const auto& GetUserThreadRunnerFactory() const { + return benchmark_.threadrunner_; + } State Run(IterationCount iters, int thread_id, internal::ThreadTimer* timer, internal::ThreadManager* manager, - internal::PerfCountersMeasurement* perf_counters_measurement) const; + internal::PerfCountersMeasurement* perf_counters_measurement, + ProfilerManager* profiler_manager) const; private: BenchmarkName name_; - Benchmark& benchmark_; + benchmark::Benchmark& benchmark_; const int family_index_; const int per_family_instance_index_; AggregationReportMode aggregation_report_mode_; @@ -67,11 +73,11 @@ class BenchmarkInstance { IterationCount iterations_; int threads_; // Number of concurrent threads to us - typedef void (*callback_function)(const benchmark::State&); - callback_function setup_ = nullptr; - callback_function teardown_ = nullptr; + callback_function setup_; + callback_function teardown_; }; +BENCHMARK_EXPORT bool FindBenchmarksInternal(const std::string& re, std::vector* benchmarks, std::ostream* Err); diff --git a/src/benchmark_main.cc b/src/benchmark_main.cc index cd61cd2ad5..05016439ec 100644 --- a/src/benchmark_main.cc +++ b/src/benchmark_main.cc @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "benchmark/benchmark.h" +#include "benchmark/export.h" +#include "benchmark/registration.h" -BENCHMARK_EXPORT int main(int, char**); +BENCHMARK_EXPORT int main(int /*argc*/, char** /*argv*/); BENCHMARK_MAIN(); diff --git a/src/benchmark_name.cc b/src/benchmark_name.cc index 4f7386068d..710eb6db4b 100644 --- a/src/benchmark_name.cc +++ b/src/benchmark_name.cc @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include "benchmark/export.h" +#include "benchmark/reporter.h" namespace benchmark { @@ -27,8 +28,8 @@ size_t size_impl(const Head& head, const Tail&... tail) { } // Join a pack of std::strings using a delimiter -// TODO: use absl::StrJoin -void join_impl(std::string&, char) {} +// TODO(dominic): use absl::StrJoin +void join_impl(std::string& /*unused*/, char /*unused*/) {} template void join_impl(std::string& s, const char delimiter, const Head& head, @@ -51,6 +52,7 @@ std::string join(char delimiter, const Ts&... ts) { } } // namespace +BENCHMARK_EXPORT std::string BenchmarkName::str() const { return join('/', function_name, args, min_time, min_warmup_time, iterations, repetitions, time_type, threads); diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index a42b76689b..560a762e9b 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -15,7 +15,8 @@ #include "benchmark_register.h" #ifndef BENCHMARK_OS_WINDOWS -#ifndef BENCHMARK_OS_FUCHSIA +#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) && \ + !defined(BENCHMARK_OS_WASI) #include #endif #include @@ -36,7 +37,11 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/reporter.h" +#include "benchmark/statistics.h" +#include "benchmark/types.h" #include "benchmark_api_internal.h" #include "check.h" #include "commandlineflags.h" @@ -53,13 +58,13 @@ namespace benchmark { namespace { // For non-dense Range, intermediate values are powers of kRangeMultiplier. -static constexpr int kRangeMultiplier = 8; +constexpr int kRangeMultiplier = 8; // The size of a benchmark family determines is the number of inputs to repeat // the benchmark on. If this is "large" then warn the user during configuration. -static constexpr size_t kMaxFamilySize = 100; +constexpr size_t kMaxFamilySize = 100; -static constexpr char kDisabledPrefix[] = "DISABLED_"; +constexpr char kDisabledPrefix[] = "DISABLED_"; } // end namespace namespace internal { @@ -75,21 +80,21 @@ class BenchmarkFamilies { static BenchmarkFamilies* GetInstance(); // Registers a benchmark family and returns the index assigned to it. - size_t AddBenchmark(std::unique_ptr family); + size_t AddBenchmark(std::unique_ptr family); // Clear all registered benchmark families. void ClearBenchmarks(); // Extract the list of benchmark instances that match the specified // regular expression. - bool FindBenchmarks(std::string re, + bool FindBenchmarks(std::string spec, std::vector* benchmarks, std::ostream* Err); private: BenchmarkFamilies() {} - std::vector> families_; + std::vector> families_; Mutex mutex_; }; @@ -98,7 +103,8 @@ BenchmarkFamilies* BenchmarkFamilies::GetInstance() { return &instance; } -size_t BenchmarkFamilies::AddBenchmark(std::unique_ptr family) { +size_t BenchmarkFamilies::AddBenchmark( + std::unique_ptr family) { MutexLock l(mutex_); size_t index = families_.size(); families_.push_back(std::move(family)); @@ -125,7 +131,7 @@ bool BenchmarkFamilies::FindBenchmarks( is_negative_filter = true; } if (!re.Init(spec, &error_msg)) { - Err << "Could not compile benchmark re: " << error_msg << std::endl; + Err << "Could not compile benchmark re: " << error_msg << '\n'; return false; } @@ -135,12 +141,14 @@ bool BenchmarkFamilies::FindBenchmarks( int next_family_index = 0; MutexLock l(mutex_); - for (std::unique_ptr& family : families_) { + for (std::unique_ptr& family : families_) { int family_index = next_family_index; int per_family_instance_index = 0; // Family was deleted or benchmark doesn't match - if (!family) continue; + if (!family) { + continue; + } if (family->ArgsCnt() == -1) { family->Args({}); @@ -159,7 +167,9 @@ bool BenchmarkFamilies::FindBenchmarks( // reserve in the special case the regex ".", since we know the final // family size. this doesn't take into account any disabled benchmarks // so worst case we reserve more than we need. - if (spec == ".") benchmarks->reserve(benchmarks->size() + family_size); + if (spec == ".") { + benchmarks->reserve(benchmarks->size() + family_size); + } for (auto const& args : family->args_) { for (int num_threads : *thread_counts) { @@ -175,9 +185,11 @@ bool BenchmarkFamilies::FindBenchmarks( ++per_family_instance_index; - // Only bump the next family index once we've estabilished that + // Only bump the next family index once we've established that // at least one instance of this family will be run. - if (next_family_index == family_index) ++next_family_index; + if (next_family_index == family_index) { + ++next_family_index; + } } } } @@ -185,11 +197,12 @@ bool BenchmarkFamilies::FindBenchmarks( return true; } -Benchmark* RegisterBenchmarkInternal(Benchmark* bench) { - std::unique_ptr bench_ptr(bench); +benchmark::Benchmark* RegisterBenchmarkInternal( + std::unique_ptr bench) { + benchmark::Benchmark* bench_ptr = bench.get(); BenchmarkFamilies* families = BenchmarkFamilies::GetInstance(); - families->AddBenchmark(std::move(bench_ptr)); - return bench; + families->AddBenchmark(std::move(bench)); + return bench_ptr; } // FIXME: This function is a hack so that benchmark.cc can access @@ -200,13 +213,15 @@ bool FindBenchmarksInternal(const std::string& re, return BenchmarkFamilies::GetInstance()->FindBenchmarks(re, benchmarks, Err); } +} // end namespace internal + //=============================================================================// // Benchmark //=============================================================================// -Benchmark::Benchmark(const char* name) +Benchmark::Benchmark(const std::string& name) : name_(name), - aggregation_report_mode_(ARM_Unspecified), + aggregation_report_mode_(internal::ARM_Unspecified), time_unit_(GetDefaultTimeUnit()), use_default_time_unit_(true), range_multiplier_(kRangeMultiplier), @@ -218,9 +233,7 @@ Benchmark::Benchmark(const char* name) use_real_time_(false), use_manual_time_(false), complexity_(oNone), - complexity_lambda_(nullptr), - setup_(nullptr), - teardown_(nullptr) { + complexity_lambda_(nullptr) { ComputeStatistics("mean", StatisticsMean); ComputeStatistics("median", StatisticsMedian); ComputeStatistics("stddev", StatisticsStdDev); @@ -230,7 +243,7 @@ Benchmark::Benchmark(const char* name) Benchmark::~Benchmark() {} Benchmark* Benchmark::Name(const std::string& name) { - SetName(name.c_str()); + SetName(name); return this; } @@ -249,7 +262,7 @@ Benchmark* Benchmark::Unit(TimeUnit unit) { Benchmark* Benchmark::Range(int64_t start, int64_t limit) { BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); std::vector arglist; - AddRange(&arglist, start, limit, range_multiplier_); + internal::AddRange(&arglist, start, limit, range_multiplier_); for (int64_t i : arglist) { args_.push_back({i}); @@ -262,8 +275,8 @@ Benchmark* Benchmark::Ranges( BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(ranges.size())); std::vector> arglists(ranges.size()); for (std::size_t i = 0; i < ranges.size(); i++) { - AddRange(&arglists[i], ranges[i].first, ranges[i].second, - range_multiplier_); + internal::AddRange(&arglists[i], ranges[i].first, ranges[i].second, + range_multiplier_); } ArgsProduct(arglists); @@ -326,18 +339,31 @@ Benchmark* Benchmark::Args(const std::vector& args) { return this; } -Benchmark* Benchmark::Apply(void (*custom_arguments)(Benchmark* benchmark)) { +Benchmark* Benchmark::Apply( + const std::function& custom_arguments) { custom_arguments(this); return this; } -Benchmark* Benchmark::Setup(void (*setup)(const benchmark::State&)) { +Benchmark* Benchmark::Setup(callback_function&& setup) { + BM_CHECK(setup != nullptr); + setup_ = std::forward(setup); + return this; +} + +Benchmark* Benchmark::Setup(const callback_function& setup) { BM_CHECK(setup != nullptr); setup_ = setup; return this; } -Benchmark* Benchmark::Teardown(void (*teardown)(const benchmark::State&)) { +Benchmark* Benchmark::Teardown(callback_function&& teardown) { + BM_CHECK(teardown != nullptr); + teardown_ = std::forward(teardown); + return this; +} + +Benchmark* Benchmark::Teardown(const callback_function& teardown) { BM_CHECK(teardown != nullptr); teardown_ = teardown; return this; @@ -365,8 +391,8 @@ Benchmark* Benchmark::MinWarmUpTime(double t) { Benchmark* Benchmark::Iterations(IterationCount n) { BM_CHECK(n > 0); - BM_CHECK(IsZero(min_time_)); - BM_CHECK(IsZero(min_warmup_time_)); + BM_CHECK(internal::IsZero(min_time_)); + BM_CHECK(internal::IsZero(min_warmup_time_)); iterations_ = n; return this; } @@ -378,21 +404,23 @@ Benchmark* Benchmark::Repetitions(int n) { } Benchmark* Benchmark::ReportAggregatesOnly(bool value) { - aggregation_report_mode_ = value ? ARM_ReportAggregatesOnly : ARM_Default; + aggregation_report_mode_ = + value ? internal::ARM_ReportAggregatesOnly : internal::ARM_Default; return this; } Benchmark* Benchmark::DisplayAggregatesOnly(bool value) { // If we were called, the report mode is no longer 'unspecified', in any case. + using internal::AggregationReportMode; aggregation_report_mode_ = static_cast( - aggregation_report_mode_ | ARM_Default); + aggregation_report_mode_ | internal::ARM_Default); if (value) { aggregation_report_mode_ = static_cast( - aggregation_report_mode_ | ARM_DisplayReportAggregatesOnly); + aggregation_report_mode_ | internal::ARM_DisplayReportAggregatesOnly); } else { aggregation_report_mode_ = static_cast( - aggregation_report_mode_ & ~ARM_DisplayReportAggregatesOnly); + aggregation_report_mode_ & ~internal::ARM_DisplayReportAggregatesOnly); } return this; @@ -446,7 +474,7 @@ Benchmark* Benchmark::ThreadRange(int min_threads, int max_threads) { BM_CHECK_GT(min_threads, 0); BM_CHECK_GE(max_threads, min_threads); - AddRange(&thread_counts_, min_threads, max_threads, 2); + internal::AddRange(&thread_counts_, min_threads, max_threads, 2); return this; } @@ -468,20 +496,38 @@ Benchmark* Benchmark::ThreadPerCpu() { return this; } -void Benchmark::SetName(const char* name) { name_ = name; } +Benchmark* Benchmark::ThreadRunner(threadrunner_factory&& factory) { + threadrunner_ = std::move(factory); + return this; +} + +void Benchmark::SetName(const std::string& name) { name_ = name; } + +const char* Benchmark::GetName() const { return name_.c_str(); } int Benchmark::ArgsCnt() const { if (args_.empty()) { - if (arg_names_.empty()) return -1; + if (arg_names_.empty()) { + return -1; + } return static_cast(arg_names_.size()); } return static_cast(args_.front().size()); } +const char* Benchmark::GetArgName(int arg) const { + BM_CHECK_GE(arg, 0); + size_t uarg = static_cast(arg); + BM_CHECK_LT(uarg, arg_names_.size()); + return arg_names_[uarg].c_str(); +} + TimeUnit Benchmark::GetTimeUnit() const { return use_default_time_unit_ ? GetDefaultTimeUnit() : time_unit_; } +namespace internal { + //=============================================================================// // FunctionBenchmark //=============================================================================// diff --git a/src/benchmark_register.h b/src/benchmark_register.h index a5a250cc70..e0ace51ef0 100644 --- a/src/benchmark_register.h +++ b/src/benchmark_register.h @@ -1,7 +1,9 @@ #ifndef BENCHMARK_REGISTER_H #define BENCHMARK_REGISTER_H +#include #include +#include #include #include "check.h" @@ -23,7 +25,7 @@ typename std::vector::iterator AddPowers(std::vector* dst, T lo, T hi, static const T kmax = std::numeric_limits::max(); // Space out the values in multiples of "mult" - for (T i = static_cast(1); i <= hi; i *= static_cast(mult)) { + for (T i = static_cast(1); i <= hi; i = static_cast(i * mult)) { if (i >= lo) { dst->push_back(i); } @@ -51,7 +53,7 @@ void AddNegatedPowers(std::vector* dst, T lo, T hi, int mult) { const auto it = AddPowers(dst, hi_complement, lo_complement, mult); - std::for_each(it, dst->end(), [](T& t) { t *= -1; }); + std::for_each(it, dst->end(), [](T& t) { t = static_cast(t * -1); }); std::reverse(it, dst->end()); } diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 0fb74bfac8..f6d37e0172 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -14,12 +14,17 @@ #include "benchmark_runner.h" -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/managers.h" +#include "benchmark/reporter.h" +#include "benchmark/state.h" +#include "benchmark/types.h" #include "benchmark_api_internal.h" #include "internal_macros.h" #ifndef BENCHMARK_OS_WINDOWS -#ifndef BENCHMARK_OS_FUCHSIA +#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) && \ + !defined(BENCHMARK_OS_WASI) #include #endif #include @@ -28,11 +33,15 @@ #include #include +#include +#include #include #include #include #include +#include #include +#include #include #include #include @@ -43,7 +52,6 @@ #include "commandlineflags.h" #include "complexity.h" #include "counter.h" -#include "internal_macros.h" #include "log.h" #include "mutex.h" #include "perf_counters.h" @@ -55,19 +63,31 @@ namespace benchmark { +BM_DECLARE_bool(benchmark_dry_run); +BM_DECLARE_string(benchmark_min_time); +BM_DECLARE_double(benchmark_min_warmup_time); +BM_DECLARE_int32(benchmark_repetitions); +BM_DECLARE_bool(benchmark_report_aggregates_only); +BM_DECLARE_bool(benchmark_display_aggregates_only); +BM_DECLARE_string(benchmark_perf_counters); + namespace internal { MemoryManager* memory_manager = nullptr; +ProfilerManager* profiler_manager = nullptr; + namespace { -static constexpr IterationCount kMaxIterations = 1000000000; +constexpr IterationCount kMaxIterations = 1000000000000; +const double kDefaultMinTime = + std::strtod(::benchmark::kDefaultMinTimeStr, /*p_end*/ nullptr); BenchmarkReporter::Run CreateRunReport( const benchmark::internal::BenchmarkInstance& b, const internal::ThreadManager::Result& results, IterationCount memory_iterations, - const MemoryManager::Result* memory_result, double seconds, + const MemoryManager::Result& memory_result, double seconds, int64_t repetition_index, int64_t repeats) { // Create report about this benchmark run. BenchmarkReporter::Run report; @@ -75,8 +95,8 @@ BenchmarkReporter::Run CreateRunReport( report.run_name = b.name(); report.family_index = b.family_index(); report.per_family_instance_index = b.per_family_instance_index(); - report.error_occurred = results.has_error_; - report.error_message = results.error_message_; + report.skipped = results.skipped_; + report.skip_message = results.skip_message_; report.report_label = results.report_label_; // This is the total iterations across all threads. report.iterations = results.iterations; @@ -85,12 +105,13 @@ BenchmarkReporter::Run CreateRunReport( report.repetition_index = repetition_index; report.repetitions = repeats; - if (!report.error_occurred) { + if (report.skipped == 0u) { if (b.use_manual_time()) { report.real_accumulated_time = results.manual_time_used; } else { report.real_accumulated_time = results.real_time_used; } + report.use_real_time_for_initial_big_o = b.use_manual_time(); report.cpu_accumulated_time = results.cpu_time_used; report.complexity_n = results.complexity_n; report.complexity = b.complexity(); @@ -99,15 +120,20 @@ BenchmarkReporter::Run CreateRunReport( report.counters = results.counters; if (memory_iterations > 0) { - assert(memory_result != nullptr); report.memory_result = memory_result; report.allocs_per_iter = - memory_iterations ? static_cast(memory_result->num_allocs) / - memory_iterations - : 0; + memory_iterations != 0 + ? static_cast(memory_result.num_allocs) / + static_cast(memory_iterations) + : 0; } - internal::Finish(&report.counters, results.iterations, seconds, + // The CPU time is the total time taken by all thread. If we used that as + // the denominator, we'd be calculating the rate per thread here. This is + // why we have to divide the total cpu_time by the number of threads for + // global counters to get a global rate. + const double thread_seconds = seconds / b.threads(); + internal::Finish(&report.counters, results.iterations, thread_seconds, b.threads()); } return report; @@ -117,16 +143,20 @@ BenchmarkReporter::Run CreateRunReport( // Adds the stats collected for the thread into manager->results. void RunInThread(const BenchmarkInstance* b, IterationCount iters, int thread_id, ThreadManager* manager, - PerfCountersMeasurement* perf_counters_measurement) { + PerfCountersMeasurement* perf_counters_measurement, + ProfilerManager* profiler_manager_) { internal::ThreadTimer timer( b->measure_process_cpu_time() ? internal::ThreadTimer::CreateProcessCpuTime() : internal::ThreadTimer::Create()); - State st = - b->Run(iters, thread_id, &timer, manager, perf_counters_measurement); - BM_CHECK(st.error_occurred() || st.iterations() >= st.max_iterations) - << "Benchmark returned before State::KeepRunning() returned false!"; + State st = b->Run(iters, thread_id, &timer, manager, + perf_counters_measurement, profiler_manager_); + if (!(st.skipped() || st.iterations() >= st.max_iterations)) { + st.SkipWithError( + "The benchmark didn't run, nor was it explicitly skipped. Please call " + "'SkipWithXXX` in your benchmark as appropriate."); + } { MutexLock l(manager->GetBenchmarkMutex()); internal::ThreadManager::Result& results = manager->results; @@ -140,27 +170,149 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, manager->NotifyThreadComplete(); } +double ComputeMinTime(const benchmark::internal::BenchmarkInstance& b, + const BenchTimeType& iters_or_time) { + if (!IsZero(b.min_time())) { + return b.min_time(); + } + // If the flag was used to specify number of iters, then return the default + // min_time. + if (iters_or_time.tag == BenchTimeType::ITERS) { + return kDefaultMinTime; + } + + return iters_or_time.time; +} + +IterationCount ComputeIters(const benchmark::internal::BenchmarkInstance& b, + const BenchTimeType& iters_or_time) { + if (b.iterations() != 0) { + return b.iterations(); + } + + // We've already concluded that this flag is currently used to pass + // iters but do a check here again anyway. + BM_CHECK(iters_or_time.tag == BenchTimeType::ITERS); + return iters_or_time.iters; +} + +class ThreadRunnerDefault : public ThreadRunnerBase { + public: + explicit ThreadRunnerDefault(int num_threads) + : pool(static_cast(num_threads - 1)) {} + + void RunThreads(const std::function& fn) override final { + // Run all but one thread in separate threads + for (std::size_t ti = 0; ti < pool.size(); ++ti) { + pool[ti] = std::thread(fn, static_cast(ti + 1)); + } + // And run one thread here directly. + // (If we were asked to run just one thread, we don't create new threads.) + // Yes, we need to do this here *after* we start the separate threads. + fn(0); + + // The main thread has finished. Now let's wait for the other threads. + for (std::thread& thread : pool) { + thread.join(); + } + } + + private: + std::vector pool; +}; + +std::unique_ptr GetThreadRunner( + const benchmark::threadrunner_factory& userThreadRunnerFactory, + int num_threads) { + return userThreadRunnerFactory + ? userThreadRunnerFactory(num_threads) + : std::make_unique(num_threads); +} + } // end namespace +BenchTimeType ParseBenchMinTime(const std::string& value) { + BenchTimeType ret = {}; + + if (value.empty()) { + ret.tag = BenchTimeType::TIME; + ret.time = 0.0; + return ret; + } + + if (value.back() == 'x') { + char* p_end = nullptr; + // Reset errno before it's changed by strtol. + errno = 0; + IterationCount num_iters = std::strtol(value.c_str(), &p_end, 10); + + // After a valid parse, p_end should have been set to + // point to the 'x' suffix. + BM_CHECK(errno == 0 && p_end != nullptr && *p_end == 'x') + << "Malformed iters value passed to --benchmark_min_time: `" << value + << "`. Expected --benchmark_min_time=x."; + + ret.tag = BenchTimeType::ITERS; + ret.iters = num_iters; + return ret; + } + + bool has_suffix = value.back() == 's'; + if (!has_suffix) { + BM_VLOG(0) << "Value passed to --benchmark_min_time should have a suffix. " + "Eg., `30s` for 30-seconds."; + } + + char* p_end = nullptr; + // Reset errno before it's changed by strtod. + errno = 0; + double min_time = std::strtod(value.c_str(), &p_end); + + // After a successful parse, p_end should point to the suffix 's', + // or the end of the string if the suffix was omitted. + BM_CHECK(errno == 0 && p_end != nullptr && + ((has_suffix && *p_end == 's') || *p_end == '\0')) + << "Malformed seconds value passed to --benchmark_min_time: `" << value + << "`. Expected --benchmark_min_time=x."; + + ret.tag = BenchTimeType::TIME; + ret.time = min_time; + + return ret; +} + BenchmarkRunner::BenchmarkRunner( const benchmark::internal::BenchmarkInstance& b_, + PerfCountersMeasurement* pcm_, BenchmarkReporter::PerFamilyRunReports* reports_for_family_) : b(b_), reports_for_family(reports_for_family_), - min_time(!IsZero(b.min_time()) ? b.min_time() : FLAGS_benchmark_min_time), - min_warmup_time((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) - ? b.min_warmup_time() - : FLAGS_benchmark_min_warmup_time), - warmup_done(!(min_warmup_time > 0.0)), - repeats(b.repetitions() != 0 ? b.repetitions() - : FLAGS_benchmark_repetitions), - has_explicit_iteration_count(b.iterations() != 0), - pool(b.threads() - 1), - iters(has_explicit_iteration_count ? b.iterations() : 1), - perf_counters_measurement(StrSplit(FLAGS_benchmark_perf_counters, ',')), - perf_counters_measurement_ptr(perf_counters_measurement.IsValid() - ? &perf_counters_measurement - : nullptr) { + parsed_benchtime_flag(ParseBenchMinTime(FLAGS_benchmark_min_time)), + min_time(FLAGS_benchmark_dry_run + ? 0 + : ComputeMinTime(b_, parsed_benchtime_flag)), + min_warmup_time( + FLAGS_benchmark_dry_run + ? 0 + : ((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) + ? b.min_warmup_time() + : FLAGS_benchmark_min_warmup_time)), + warmup_done(FLAGS_benchmark_dry_run ? true : !(min_warmup_time > 0.0)), + repeats(FLAGS_benchmark_dry_run + ? 1 + : (b.repetitions() != 0 ? b.repetitions() + : FLAGS_benchmark_repetitions)), + has_explicit_iteration_count(b.iterations() != 0 || + parsed_benchtime_flag.tag == + BenchTimeType::ITERS), + thread_runner( + GetThreadRunner(b.GetUserThreadRunnerFactory(), b.threads())), + iters(FLAGS_benchmark_dry_run + ? 1 + : (has_explicit_iteration_count + ? ComputeIters(b_, parsed_benchtime_flag) + : 1)), + perf_counters_measurement_ptr(pcm_) { run_results.display_report_aggregates_only = (FLAGS_benchmark_report_aggregates_only || FLAGS_benchmark_display_aggregates_only); @@ -168,12 +320,13 @@ BenchmarkRunner::BenchmarkRunner( FLAGS_benchmark_report_aggregates_only; if (b.aggregation_report_mode() != internal::ARM_Unspecified) { run_results.display_report_aggregates_only = - (b.aggregation_report_mode() & - internal::ARM_DisplayReportAggregatesOnly); + ((b.aggregation_report_mode() & + internal::ARM_DisplayReportAggregatesOnly) != 0u); run_results.file_report_aggregates_only = - (b.aggregation_report_mode() & internal::ARM_FileReportAggregatesOnly); + ((b.aggregation_report_mode() & + internal::ARM_FileReportAggregatesOnly) != 0u); BM_CHECK(FLAGS_benchmark_perf_counters.empty() || - perf_counters_measurement.IsValid()) + (perf_counters_measurement_ptr->num_counters() == 0)) << "Perf counters were requested but could not be set up."; } } @@ -184,19 +337,10 @@ BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() { std::unique_ptr manager; manager.reset(new internal::ThreadManager(b.threads())); - // Run all but one thread in separate threads - for (std::size_t ti = 0; ti < pool.size(); ++ti) { - pool[ti] = std::thread(&RunInThread, &b, iters, static_cast(ti + 1), - manager.get(), perf_counters_measurement_ptr); - } - // And run one thread here directly. - // (If we were asked to run just one thread, we don't create new threads.) - // Yes, we need to do this here *after* we start the separate threads. - RunInThread(&b, iters, 0, manager.get(), perf_counters_measurement_ptr); - - // The main thread has finished. Now let's wait for the other threads. - manager->WaitForAllThreads(); - for (std::thread& thread : pool) thread.join(); + thread_runner->RunThreads([&](int thread_idx) { + RunInThread(&b, iters, thread_idx, manager.get(), + perf_counters_measurement_ptr, /*profiler_manager=*/nullptr); + }); IterationResults i; // Acquire the measurements/counters from the manager, UNDER THE LOCK! @@ -208,12 +352,6 @@ BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() { // And get rid of the manager. manager.reset(); - // Adjust real/manual time stats since they were reported per thread. - i.results.real_time_used /= b.threads(); - i.results.manual_time_used /= b.threads(); - // If we were measuring whole-process CPU usage, adjust the CPU time too. - if (b.measure_process_cpu_time()) i.results.cpu_time_used /= b.threads(); - BM_VLOG(2) << "Ran in " << i.results.cpu_time_used << "/" << i.results.real_time_used << "\n"; @@ -247,8 +385,8 @@ IterationCount BenchmarkRunner::PredictNumItersNeeded( // So what seems to be the sufficiently-large iteration count? Round up. const IterationCount max_next_iters = static_cast( - std::lround(std::max(multiplier * static_cast(i.iters), - static_cast(i.iters) + 1.0))); + std::llround(std::max(multiplier * static_cast(i.iters), + static_cast(i.iters) + 1.0))); // But we do have *some* limits though.. const IterationCount next_iters = std::min(max_next_iters, kMaxIterations); @@ -261,7 +399,7 @@ bool BenchmarkRunner::ShouldReportIterationResults( // Determine if this run should be reported; // Either it has run for a sufficient amount of time // or because an error was reported. - return i.results.has_error_ || + return (i.results.skipped_ != 0u) || FLAGS_benchmark_dry_run || i.iters >= kMaxIterations || // Too many iterations already. i.seconds >= GetMinTimeToApply() || // The elapsed time is large enough. @@ -273,7 +411,7 @@ bool BenchmarkRunner::ShouldReportIterationResults( } double BenchmarkRunner::GetMinTimeToApply() const { - // In order to re-use functionality to run and measure benchmarks for running + // In order to reuse functionality to run and measure benchmarks for running // a warmup phase of the benchmark, we need a way of telling whether to apply // min_time or min_warmup_time. This function will figure out if we are in the // warmup phase and therefore need to apply min_warmup_time or if we already @@ -322,6 +460,34 @@ void BenchmarkRunner::RunWarmUp() { } } +MemoryManager::Result BenchmarkRunner::RunMemoryManager( + IterationCount memory_iterations) { + memory_manager->Start(); + std::unique_ptr manager; + manager.reset(new internal::ThreadManager(1)); + b.Setup(); + RunInThread(&b, memory_iterations, 0, manager.get(), + perf_counters_measurement_ptr, + /*profiler_manager=*/nullptr); + manager.reset(); + b.Teardown(); + MemoryManager::Result memory_result; + memory_manager->Stop(memory_result); + memory_result.memory_iterations = memory_iterations; + return memory_result; +} + +void BenchmarkRunner::RunProfilerManager(IterationCount profile_iterations) { + std::unique_ptr manager; + manager.reset(new internal::ThreadManager(1)); + b.Setup(); + RunInThread(&b, profile_iterations, 0, manager.get(), + /*perf_counters_measurement_ptr=*/nullptr, + /*profiler_manager=*/profiler_manager); + manager.reset(); + b.Teardown(); +} + void BenchmarkRunner::DoOneRepetition() { assert(HasRepeatsRemaining() && "Already done all repetitions?"); @@ -332,7 +498,9 @@ void BenchmarkRunner::DoOneRepetition() { // this warmup never happened except the fact that warmup_done is set. Every // other manipulation of the BenchmarkRunner instance would be a bug! Please // fix it. - if (!warmup_done) RunWarmUp(); + if (!warmup_done) { + RunWarmUp(); + } IterationResults i; // We *may* be gradually increasing the length (iteration count) @@ -354,8 +522,10 @@ void BenchmarkRunner::DoOneRepetition() { const bool results_are_significant = !is_the_first_repetition || has_explicit_iteration_count || ShouldReportIterationResults(i); - - if (results_are_significant) break; // Good, let's report them! + // Good, let's report them! + if (results_are_significant) { + break; + } // Nope, bad iteration. Let's re-estimate the hopefully-sufficient // iteration count, and run the benchmark again... @@ -366,31 +536,21 @@ void BenchmarkRunner::DoOneRepetition() { "then we should have accepted the current iteration run."); } - // Oh, one last thing, we need to also produce the 'memory measurements'.. - MemoryManager::Result* memory_result = nullptr; + // Produce memory measurements if requested. + MemoryManager::Result memory_result; IterationCount memory_iterations = 0; if (memory_manager != nullptr) { - // TODO(vyng): Consider making BenchmarkReporter::Run::memory_result an - // optional so we don't have to own the Result here. - // Can't do it now due to cxx03. - memory_results.push_back(MemoryManager::Result()); - memory_result = &memory_results.back(); // Only run a few iterations to reduce the impact of one-time // allocations in benchmarks that are not properly managed. memory_iterations = std::min(16, iters); - memory_manager->Start(); - std::unique_ptr manager; - manager.reset(new internal::ThreadManager(1)); - b.Setup(); - RunInThread(&b, memory_iterations, 0, manager.get(), - perf_counters_measurement_ptr); - manager->WaitForAllThreads(); - manager.reset(); - b.Teardown(); + memory_result = RunMemoryManager(memory_iterations); + } - BENCHMARK_DISABLE_DEPRECATED_WARNING - memory_manager->Stop(memory_result); - BENCHMARK_RESTORE_DEPRECATED_WARNING + if (profiler_manager != nullptr) { + // We want to externally profile the benchmark for the same number of + // iterations because, for example, if we're tracing the benchmark then we + // want trace data to reasonably match PMU data. + RunProfilerManager(iters); } // Ok, now actually report. @@ -398,9 +558,11 @@ void BenchmarkRunner::DoOneRepetition() { CreateRunReport(b, i.results, memory_iterations, memory_result, i.seconds, num_repetitions_done, repeats); - if (reports_for_family) { + if (reports_for_family != nullptr) { ++reports_for_family->num_runs_done; - if (!report.error_occurred) reports_for_family->Runs.push_back(report); + if (report.skipped == 0u) { + reports_for_family->Runs.push_back(report); + } } run_results.non_aggregates.push_back(report); diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 0174bd3401..9a2231a2a4 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -15,26 +15,20 @@ #ifndef BENCHMARK_RUNNER_H_ #define BENCHMARK_RUNNER_H_ +#include #include #include #include "benchmark_api_internal.h" -#include "internal_macros.h" #include "perf_counters.h" #include "thread_manager.h" namespace benchmark { -BM_DECLARE_double(benchmark_min_time); -BM_DECLARE_double(benchmark_min_warmup_time); -BM_DECLARE_int32(benchmark_repetitions); -BM_DECLARE_bool(benchmark_report_aggregates_only); -BM_DECLARE_bool(benchmark_display_aggregates_only); -BM_DECLARE_string(benchmark_perf_counters); - namespace internal { extern MemoryManager* memory_manager; +extern ProfilerManager* profiler_manager; struct RunResults { std::vector non_aggregates; @@ -44,9 +38,21 @@ struct RunResults { bool file_report_aggregates_only = false; }; +struct BENCHMARK_EXPORT BenchTimeType { + enum { UNSPECIFIED, ITERS, TIME } tag; + union { + IterationCount iters; + double time; + }; +}; + +BENCHMARK_EXPORT +BenchTimeType ParseBenchMinTime(const std::string& value); + class BenchmarkRunner { public: BenchmarkRunner(const benchmark::internal::BenchmarkInstance& b_, + benchmark::internal::PerfCountersMeasurement* pcm_, BenchmarkReporter::PerFamilyRunReports* reports_for_family); int GetNumRepeats() const { return repeats; } @@ -63,12 +69,19 @@ class BenchmarkRunner { return reports_for_family; } + double GetMinTime() const { return min_time; } + + bool HasExplicitIters() const { return has_explicit_iteration_count; } + + IterationCount GetIters() const { return iters; } + private: RunResults run_results; const benchmark::internal::BenchmarkInstance& b; BenchmarkReporter::PerFamilyRunReports* reports_for_family; + BenchTimeType parsed_benchtime_flag; const double min_time; const double min_warmup_time; bool warmup_done; @@ -77,16 +90,13 @@ class BenchmarkRunner { int num_repetitions_done = 0; - std::vector pool; - - std::vector memory_results; + std::unique_ptr thread_runner; IterationCount iters; // preserved between repetitions! // So only the first repetition has to find/calculate it, // the other repetitions will just use that precomputed iteration count. - PerfCountersMeasurement perf_counters_measurement; - PerfCountersMeasurement* const perf_counters_measurement_ptr; + PerfCountersMeasurement* const perf_counters_measurement_ptr = nullptr; struct IterationResults { internal::ThreadManager::Result results; @@ -95,6 +105,10 @@ class BenchmarkRunner { }; IterationResults DoNIterations(); + MemoryManager::Result RunMemoryManager(IterationCount memory_iterations); + + void RunProfilerManager(IterationCount profile_iterations); + IterationCount PredictNumItersNeeded(const IterationResults& i) const; bool ShouldReportIterationResults(const IterationResults& i) const; diff --git a/src/check.cc b/src/check.cc index 422b9483a8..3e2a40b4bd 100644 --- a/src/check.cc +++ b/src/check.cc @@ -3,9 +3,12 @@ namespace benchmark { namespace internal { -static AbortHandlerT* handler = &std::abort; +namespace { +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +AbortHandlerT* handler = &std::abort; +} // namespace -AbortHandlerT*& GetAbortHandler() { return handler; } +BENCHMARK_EXPORT AbortHandlerT*& GetAbortHandler() { return handler; } } // namespace internal } // namespace benchmark diff --git a/src/check.h b/src/check.h index 1129e81402..aa8c78c92f 100644 --- a/src/check.h +++ b/src/check.h @@ -4,11 +4,29 @@ #include #include #include +#include #include "benchmark/export.h" #include "internal_macros.h" #include "log.h" +#if defined(__GNUC__) || defined(__clang__) +#define BENCHMARK_NOEXCEPT noexcept +#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) +#elif defined(_MSC_VER) && !defined(__clang__) +#if _MSC_VER >= 1900 +#define BENCHMARK_NOEXCEPT noexcept +#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) +#else +#define BENCHMARK_NOEXCEPT +#define BENCHMARK_NOEXCEPT_OP(x) +#endif +#define __func__ __FUNCTION__ +#else +#define BENCHMARK_NOEXCEPT +#define BENCHMARK_NOEXCEPT_OP(x) +#endif + namespace benchmark { namespace internal { @@ -19,6 +37,8 @@ AbortHandlerT*& GetAbortHandler(); BENCHMARK_NORETURN inline void CallAbortHandler() { GetAbortHandler()(); + std::flush(std::cout); + std::flush(std::cerr); std::abort(); // fallback to enforce noreturn } @@ -27,7 +47,8 @@ BENCHMARK_NORETURN inline void CallAbortHandler() { // destructed. class CheckHandler { public: - CheckHandler(const char* check, const char* file, const char* func, int line) + CheckHandler(std::string_view check, std::string_view file, + std::string_view func, int line) : log_(GetErrorLogInstance()) { log_ << file << ":" << line << ": " << func << ": Check `" << check << "' failed. "; @@ -40,7 +61,7 @@ class CheckHandler { #pragma warning(disable : 4722) #endif BENCHMARK_NORETURN ~CheckHandler() BENCHMARK_NOEXCEPT_OP(false) { - log_ << std::endl; + log_ << '\n'; CallAbortHandler(); } #if defined(COMPILER_MSVC) @@ -61,9 +82,11 @@ class CheckHandler { // The BM_CHECK macro returns a std::ostream object that can have extra // information written to it. #ifndef NDEBUG -#define BM_CHECK(b) \ - (b ? ::benchmark::internal::GetNullLogInstance() \ - : ::benchmark::internal::CheckHandler(#b, __FILE__, __func__, __LINE__) \ +#define BM_CHECK(b) \ + (b ? ::benchmark::internal::GetNullLogInstance() \ + : ::benchmark::internal::CheckHandler( \ + std::string_view(#b), std::string_view(__FILE__), \ + std::string_view(__func__), __LINE__) \ .GetLog()) #else #define BM_CHECK(b) ::benchmark::internal::GetNullLogInstance() diff --git a/src/colorprint.cc b/src/colorprint.cc index 1a000a0637..b7a6346442 100644 --- a/src/colorprint.cc +++ b/src/colorprint.cc @@ -96,18 +96,21 @@ std::string FormatString(const char* msg, va_list args) { // currently there is no error handling for failure, so this is hack. BM_CHECK(ret >= 0); - if (ret == 0) // handle empty expansion + if (ret == 0) { // handle empty expansion return {}; - else if (static_cast(ret) < size) + } + if (static_cast(ret) < size) { return local_buff; - else { - // we did not provide a long enough buffer on our first attempt. - size = static_cast(ret) + 1; // + 1 for the null byte - std::unique_ptr buff(new char[size]); - ret = vsnprintf(buff.get(), size, msg, args); - BM_CHECK(ret > 0 && (static_cast(ret)) < size); - return buff.get(); } + // we did not provide a long enough buffer on our first attempt. + size = static_cast(ret) + 1; // + 1 for the null byte + std::unique_ptr buff(new char[size]); + va_list args_cp2; + va_copy(args_cp2, args); + ret = vsnprintf(buff.get(), size, msg, args_cp2); + va_end(args_cp2); + BM_CHECK(ret > 0 && (static_cast(ret)) < size); + return buff.get(); } std::string FormatString(const char* msg, ...) { @@ -135,22 +138,30 @@ void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, // Gets the current text color. CONSOLE_SCREEN_BUFFER_INFO buffer_info; GetConsoleScreenBufferInfo(stdout_handle, &buffer_info); - const WORD old_color_attrs = buffer_info.wAttributes; + const WORD original_color_attrs = buffer_info.wAttributes; // We need to flush the stream buffers into the console before each // SetConsoleTextAttribute call lest it affect the text that is already // printed but has not yet reached the console. - fflush(stdout); - SetConsoleTextAttribute(stdout_handle, - GetPlatformColorCode(color) | FOREGROUND_INTENSITY); - vprintf(fmt, args); - - fflush(stdout); - // Restores the text color. - SetConsoleTextAttribute(stdout_handle, old_color_attrs); + out.flush(); + + const WORD original_background_attrs = + original_color_attrs & (BACKGROUND_RED | BACKGROUND_GREEN | + BACKGROUND_BLUE | BACKGROUND_INTENSITY); + + SetConsoleTextAttribute(stdout_handle, GetPlatformColorCode(color) | + FOREGROUND_INTENSITY | + original_background_attrs); + out << FormatString(fmt, args); + + out.flush(); + // Restores the text and background color. + SetConsoleTextAttribute(stdout_handle, original_color_attrs); #else const char* color_code = GetPlatformColorCode(color); - if (color_code) out << FormatString("\033[0;3%sm", color_code); + if (color_code != nullptr) { + out << FormatString("\033[0;3%sm", color_code); + } out << FormatString(fmt, args) << "\033[m"; #endif } @@ -163,19 +174,31 @@ bool IsColorTerminal() { #else // On non-Windows platforms, we rely on the TERM variable. This list of // supported TERM values is copied from Google Test: - // . + // . const char* const SUPPORTED_TERM_VALUES[] = { - "xterm", "xterm-color", "xterm-256color", - "screen", "screen-256color", "tmux", - "tmux-256color", "rxvt-unicode", "rxvt-unicode-256color", - "linux", "cygwin", + "xterm", + "xterm-color", + "xterm-256color", + "screen", + "screen-256color", + "tmux", + "tmux-256color", + "rxvt-unicode", + "rxvt-unicode-256color", + "linux", + "cygwin", + "xterm-kitty", + "alacritty", + "foot", + "foot-extra", + "wezterm", }; const char* const term = getenv("TERM"); bool term_supports_color = false; for (const char* candidate : SUPPORTED_TERM_VALUES) { - if (term && 0 == strcmp(term, candidate)) { + if ((term != nullptr) && 0 == strcmp(term, candidate)) { term_supports_color = true; break; } diff --git a/src/colorprint.h b/src/colorprint.h index 9f6fab9b34..469045c5f5 100644 --- a/src/colorprint.h +++ b/src/colorprint.h @@ -5,6 +5,8 @@ #include #include +#include "internal_macros.h" + namespace benchmark { enum LogColor { COLOR_DEFAULT, @@ -17,11 +19,14 @@ enum LogColor { COLOR_WHITE }; +PRINTF_FORMAT_STRING_FUNC(1, 0) std::string FormatString(const char* msg, va_list args); -std::string FormatString(const char* msg, ...); +PRINTF_FORMAT_STRING_FUNC(1, 2) std::string FormatString(const char* msg, ...); +PRINTF_FORMAT_STRING_FUNC(3, 0) void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, va_list args); +PRINTF_FORMAT_STRING_FUNC(3, 4) void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, ...); // Returns true if stdout appears to be a terminal that supports colored diff --git a/src/commandlineflags.cc b/src/commandlineflags.cc index 9615e351ff..99a240c122 100644 --- a/src/commandlineflags.cc +++ b/src/commandlineflags.cc @@ -109,24 +109,27 @@ bool ParseKvPairs(const std::string& src_text, const char* str, // Returns the name of the environment variable corresponding to the // given flag. For example, FlagToEnvVar("foo") will return // "BENCHMARK_FOO" in the open-source version. -static std::string FlagToEnvVar(const char* flag) { +std::string FlagToEnvVar(const char* flag) { const std::string flag_str(flag); std::string env_var; - for (size_t i = 0; i != flag_str.length(); ++i) + for (size_t i = 0; i != flag_str.length(); ++i) { env_var += static_cast(::toupper(flag_str.c_str()[i])); + } return env_var; } } // namespace +BENCHMARK_EXPORT bool BoolFromEnv(const char* flag, bool default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value_str = getenv(env_var.c_str()); return value_str == nullptr ? default_val : IsTruthyFlagValue(value_str); } +BENCHMARK_EXPORT int32_t Int32FromEnv(const char* flag, int32_t default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value_str = getenv(env_var.c_str()); @@ -139,6 +142,7 @@ int32_t Int32FromEnv(const char* flag, int32_t default_val) { return value; } +BENCHMARK_EXPORT double DoubleFromEnv(const char* flag, double default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value_str = getenv(env_var.c_str()); @@ -151,18 +155,22 @@ double DoubleFromEnv(const char* flag, double default_val) { return value; } +BENCHMARK_EXPORT const char* StringFromEnv(const char* flag, const char* default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value = getenv(env_var.c_str()); return value == nullptr ? default_val : value; } +BENCHMARK_EXPORT std::map KvPairsFromEnv( const char* flag, std::map default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value_str = getenv(env_var.c_str()); - if (value_str == nullptr) return default_val; + if (value_str == nullptr) { + return default_val; + } std::map value; if (!ParseKvPairs("Environment variable " + env_var, value_str, &value)) { @@ -171,6 +179,8 @@ std::map KvPairsFromEnv( return value; } +namespace { + // Parses a string as a command line flag. The string should have // the format "--flag=value". When def_optional is true, the "=value" // part can be omitted. @@ -179,107 +189,137 @@ std::map KvPairsFromEnv( const char* ParseFlagValue(const char* str, const char* flag, bool def_optional) { // str and flag must not be nullptr. - if (str == nullptr || flag == nullptr) return nullptr; + if (str == nullptr || flag == nullptr) { + return nullptr; + } // The flag must start with "--". const std::string flag_str = std::string("--") + std::string(flag); const size_t flag_len = flag_str.length(); - if (strncmp(str, flag_str.c_str(), flag_len) != 0) return nullptr; + if (strncmp(str, flag_str.c_str(), flag_len) != 0) { + return nullptr; + } // Skips the flag name. const char* flag_end = str + flag_len; // When def_optional is true, it's OK to not have a "=value" part. - if (def_optional && (flag_end[0] == '\0')) return flag_end; + if (def_optional && (flag_end[0] == '\0')) { + return flag_end; + } // If def_optional is true and there are more characters after the // flag name, or if def_optional is false, there must be a '=' after // the flag name. - if (flag_end[0] != '=') return nullptr; + if (flag_end[0] != '=') { + return nullptr; + } // Returns the string after "=". return flag_end + 1; } +} // end namespace + +BENCHMARK_EXPORT bool ParseBoolFlag(const char* str, const char* flag, bool* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, true); // Aborts if the parsing failed. - if (value_str == nullptr) return false; + if (value_str == nullptr) { + return false; + } // Converts the string value to a bool. *value = IsTruthyFlagValue(value_str); return true; } +BENCHMARK_EXPORT bool ParseInt32Flag(const char* str, const char* flag, int32_t* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, false); // Aborts if the parsing failed. - if (value_str == nullptr) return false; + if (value_str == nullptr) { + return false; + } // Sets *value to the value of the flag. return ParseInt32(std::string("The value of flag --") + flag, value_str, value); } +BENCHMARK_EXPORT bool ParseDoubleFlag(const char* str, const char* flag, double* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, false); // Aborts if the parsing failed. - if (value_str == nullptr) return false; + if (value_str == nullptr) { + return false; + } // Sets *value to the value of the flag. return ParseDouble(std::string("The value of flag --") + flag, value_str, value); } +BENCHMARK_EXPORT bool ParseStringFlag(const char* str, const char* flag, std::string* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, false); // Aborts if the parsing failed. - if (value_str == nullptr) return false; + if (value_str == nullptr) { + return false; + } *value = value_str; return true; } +BENCHMARK_EXPORT bool ParseKeyValueFlag(const char* str, const char* flag, std::map* value) { const char* const value_str = ParseFlagValue(str, flag, false); - if (value_str == nullptr) return false; + if (value_str == nullptr) { + return false; + } for (const auto& kvpair : StrSplit(value_str, ',')) { const auto kv = StrSplit(kvpair, '='); - if (kv.size() != 2) return false; + if (kv.size() != 2) { + return false; + } value->emplace(kv[0], kv[1]); } return true; } +BENCHMARK_EXPORT bool IsFlag(const char* str, const char* flag) { return (ParseFlagValue(str, flag, true) != nullptr); } +BENCHMARK_EXPORT bool IsTruthyFlagValue(const std::string& value) { if (value.size() == 1) { char v = value[0]; return isalnum(v) && !(v == '0' || v == 'f' || v == 'F' || v == 'n' || v == 'N'); - } else if (!value.empty()) { + } + if (!value.empty()) { std::string value_lower(value); std::transform(value_lower.begin(), value_lower.end(), value_lower.begin(), [](char c) { return static_cast(::tolower(c)); }); return !(value_lower == "false" || value_lower == "no" || value_lower == "off"); - } else - return true; + } + return true; } } // end namespace benchmark diff --git a/src/commandlineflags.h b/src/commandlineflags.h index 7882628975..5f9ebf1d56 100644 --- a/src/commandlineflags.h +++ b/src/commandlineflags.h @@ -11,14 +11,17 @@ #define FLAG(name) FLAGS_##name // Macros for declaring flags. +// NOLINTBEGIN(cppcoreguidelines-avoid-non-const-global-variables) #define BM_DECLARE_bool(name) BENCHMARK_EXPORT extern bool FLAG(name) #define BM_DECLARE_int32(name) BENCHMARK_EXPORT extern int32_t FLAG(name) #define BM_DECLARE_double(name) BENCHMARK_EXPORT extern double FLAG(name) #define BM_DECLARE_string(name) BENCHMARK_EXPORT extern std::string FLAG(name) #define BM_DECLARE_kvpairs(name) \ BENCHMARK_EXPORT extern std::map FLAG(name) +// NOLINTEND(cppcoreguidelines-avoid-non-const-global-variables) // Macros for defining flags. +// NOLINTBEGIN(cppcoreguidelines-avoid-non-const-global-variables) #define BM_DEFINE_bool(name, default_val) \ BENCHMARK_EXPORT bool FLAG(name) = benchmark::BoolFromEnv(#name, default_val) #define BM_DEFINE_int32(name, default_val) \ @@ -33,6 +36,7 @@ #define BM_DEFINE_kvpairs(name, default_val) \ BENCHMARK_EXPORT std::map FLAG(name) = \ benchmark::KvPairsFromEnv(#name, default_val) +// NOLINTEND(cppcoreguidelines-avoid-non-const-global-variables) namespace benchmark { diff --git a/src/complexity.cc b/src/complexity.cc index 825c57394a..8fa3f073af 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -17,17 +17,19 @@ #include "complexity.h" -#include #include -#include "benchmark/benchmark.h" +#include "benchmark/reporter.h" +#include "benchmark/statistics.h" +#include "benchmark/types.h" #include "check.h" namespace benchmark { +namespace { + // Internal function to calculate the different scalability forms BigOFunc* FittingCurve(BigO complexity) { - static const double kLog2E = 1.44269504088896340736; switch (complexity) { case oN: return [](IterationCount n) -> double { return static_cast(n); }; @@ -36,13 +38,12 @@ BigOFunc* FittingCurve(BigO complexity) { case oNCubed: return [](IterationCount n) -> double { return std::pow(n, 3); }; case oLogN: - /* Note: can't use log2 because Android's GNU STL lacks it */ - return - [](IterationCount n) { return kLog2E * log(static_cast(n)); }; + return [](IterationCount n) -> double { + return std::log2(static_cast(n)); + }; case oNLogN: - /* Note: can't use log2 because Android's GNU STL lacks it */ - return [](IterationCount n) { - return kLog2E * n * log(static_cast(n)); + return [](IterationCount n) -> double { + return static_cast(n) * std::log2(static_cast(n)); }; case o1: default: @@ -50,6 +51,8 @@ BigOFunc* FittingCurve(BigO complexity) { } } +} // end namespace + // Function to return an string for the calculated complexity std::string GetBigOString(BigO complexity) { switch (complexity) { @@ -70,17 +73,19 @@ std::string GetBigOString(BigO complexity) { } } +namespace { + // Find the coefficient for the high-order term in the running time, by // minimizing the sum of squares of relative error, for the fitting curve // given by the lambda expression. // - n : Vector containing the size of the benchmark tests. // - time : Vector containing the times for the benchmark tests. -// - fitting_curve : lambda expression (e.g. [](int64_t n) {return n; };). +// - fitting_curve : lambda expression (e.g. [](ComplexityN n) {return n; };). // For a deeper explanation on the algorithm logic, please refer to // https://en.wikipedia.org/wiki/Least_squares#Least_squares,_regression_analysis_and_statistics -LeastSq MinimalLeastSq(const std::vector& n, +LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, BigOFunc* fitting_curve) { double sigma_gn_squared = 0.0; @@ -105,12 +110,12 @@ LeastSq MinimalLeastSq(const std::vector& n, double rms = 0.0; for (size_t i = 0; i < n.size(); ++i) { double fit = result.coef * fitting_curve(n[i]); - rms += pow((time[i] - fit), 2); + rms += std::pow((time[i] - fit), 2); } // Normalized RMS by the mean of the observed values - double mean = sigma_time / n.size(); - result.rms = sqrt(rms / n.size()) / mean; + double mean = sigma_time / static_cast(n.size()); + result.rms = std::sqrt(rms / static_cast(n.size())) / mean; return result; } @@ -122,7 +127,7 @@ LeastSq MinimalLeastSq(const std::vector& n, // - complexity : If different than oAuto, the fitting curve will stick to // this one. If it is oAuto, it will be calculated the best // fitting curve. -LeastSq MinimalLeastSq(const std::vector& n, +LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, const BigO complexity) { BM_CHECK_EQ(n.size(), time.size()); BM_CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two @@ -154,15 +159,19 @@ LeastSq MinimalLeastSq(const std::vector& n, return best_fit; } +} // end namespace + std::vector ComputeBigO( const std::vector& reports) { typedef BenchmarkReporter::Run Run; std::vector results; - if (reports.size() < 2) return results; + if (reports.size() < 2) { + return results; + } // Accumulators. - std::vector n; + std::vector n; std::vector real_time; std::vector cpu_time; @@ -171,8 +180,10 @@ std::vector ComputeBigO( BM_CHECK_GT(run.complexity_n, 0) << "Did you forget to call SetComplexityN?"; n.push_back(run.complexity_n); - real_time.push_back(run.real_accumulated_time / run.iterations); - cpu_time.push_back(run.cpu_accumulated_time / run.iterations); + real_time.push_back(run.real_accumulated_time / + static_cast(run.iterations)); + cpu_time.push_back(run.cpu_accumulated_time / + static_cast(run.iterations)); } LeastSq result_cpu; @@ -182,8 +193,19 @@ std::vector ComputeBigO( result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity_lambda); result_real = MinimalLeastSq(n, real_time, reports[0].complexity_lambda); } else { - result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity); - result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); + const BigO* InitialBigO = &reports[0].complexity; + const bool use_real_time_for_initial_big_o = + reports[0].use_real_time_for_initial_big_o; + if (use_real_time_for_initial_big_o) { + result_real = MinimalLeastSq(n, real_time, *InitialBigO); + InitialBigO = &result_real.complexity; + // The Big-O complexity for CPU time must have the same Big-O function! + } + result_cpu = MinimalLeastSq(n, cpu_time, *InitialBigO); + InitialBigO = &result_cpu.complexity; + if (!use_real_time_for_initial_big_o) { + result_real = MinimalLeastSq(n, real_time, *InitialBigO); + } } // Drop the 'args' when reporting complexity. diff --git a/src/complexity.h b/src/complexity.h index df29b48d29..06002be215 100644 --- a/src/complexity.h +++ b/src/complexity.h @@ -21,7 +21,8 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/reporter.h" +#include "benchmark/statistics.h" namespace benchmark { @@ -31,7 +32,7 @@ std::vector ComputeBigO( const std::vector& reports); // This data structure will contain the result returned by MinimalLeastSq -// - coef : Estimated coeficient for the high-order term as +// - coef : Estimated coefficient for the high-order term as // interpolated from data. // - rms : Normalized Root Mean Squared Error. // - complexity : Scalability form (e.g. oN, oNLogN). In case a scalability diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 1711356b9b..84fe99dad1 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -21,7 +21,9 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/export.h" +#include "benchmark/reporter.h" +#include "benchmark/types.h" #include "check.h" #include "colorprint.h" #include "commandlineflags.h" @@ -33,6 +35,7 @@ namespace benchmark { +BENCHMARK_EXPORT bool ConsoleReporter::ReportContext(const Context& context) { name_field_width_ = context.name_field_width; printed_header_ = false; @@ -41,23 +44,28 @@ bool ConsoleReporter::ReportContext(const Context& context) { PrintBasicContext(&GetErrorStream(), context); #ifdef BENCHMARK_OS_WINDOWS - if ((output_options_ & OO_Color) && &std::cout != &GetOutputStream()) { - GetErrorStream() - << "Color printing is only supported for stdout on windows." - " Disabling color printing\n"; - output_options_ = static_cast(output_options_ & ~OO_Color); + if ((output_options_ & OO_Color)) { + auto stdOutBuf = std::cout.rdbuf(); + auto outStreamBuf = GetOutputStream().rdbuf(); + if (stdOutBuf != outStreamBuf) { + GetErrorStream() + << "Color printing is only supported for stdout on windows." + " Disabling color printing\n"; + output_options_ = static_cast(output_options_ & ~OO_Color); + } } #endif return true; } +BENCHMARK_EXPORT void ConsoleReporter::PrintHeader(const Run& run) { std::string str = FormatString("%-*s %13s %15s %12s", static_cast(name_field_width_), "Benchmark", "Time", "CPU", "Iterations"); if (!run.counters.empty()) { - if (output_options_ & OO_Tabular) { + if ((output_options_ & OO_Tabular) != 0) { for (auto const& c : run.counters) { str += FormatString(" %10s", c.first.c_str()); } @@ -69,6 +77,7 @@ void ConsoleReporter::PrintHeader(const Run& run) { GetOutputStream() << line << "\n" << str << "\n" << line << "\n"; } +BENCHMARK_EXPORT void ConsoleReporter::ReportRuns(const std::vector& reports) { for (const auto& run : reports) { // print the header: @@ -76,7 +85,7 @@ void ConsoleReporter::ReportRuns(const std::vector& reports) { bool print_header = !printed_header_; // --- or if the format is tabular and this run // has different fields from the prev header - print_header |= (output_options_ & OO_Tabular) && + print_header |= ((output_options_ & OO_Tabular) != 0) && (!internal::SameNames(run.counters, prev_counters_)); if (print_header) { printed_header_ = true; @@ -90,8 +99,9 @@ void ConsoleReporter::ReportRuns(const std::vector& reports) { } } -static void IgnoreColorPrint(std::ostream& out, LogColor, const char* fmt, - ...) { +PRINTF_FORMAT_STRING_FUNC(3, 4) +static void IgnoreColorPrint(std::ostream& out, LogColor /*unused*/, + const char* fmt, ...) { va_list args; va_start(args, fmt); out << FormatString(fmt, args); @@ -112,7 +122,7 @@ static std::string FormatTime(double time) { if (time < 100.0) { return FormatString("%10.1f", time); } - // Assuming the time ist at max 9.9999e+99 and we have 10 digits for the + // Assuming the time is at max 9.9999e+99 and we have 10 digits for the // number, we get 10-1(.)-1(e)-1(sign)-2(exponent) = 5 digits to print. if (time > 9999999999 /*max 10 digit number*/) { return FormatString("%1.4e", time); @@ -120,20 +130,26 @@ static std::string FormatTime(double time) { return FormatString("%10.0f", time); } +BENCHMARK_EXPORT void ConsoleReporter::PrintRunData(const Run& result) { typedef void(PrinterFn)(std::ostream&, LogColor, const char*, ...); auto& Out = GetOutputStream(); - PrinterFn* printer = (output_options_ & OO_Color) + PrinterFn* printer = (output_options_ & OO_Color) != 0 ? static_cast(ColorPrintf) : IgnoreColorPrint; auto name_color = (result.report_big_o || result.report_rms) ? COLOR_BLUE : COLOR_GREEN; - printer(Out, name_color, "%-*s ", name_field_width_, + printer(Out, name_color, "%-*s ", static_cast(name_field_width_), result.benchmark_name().c_str()); - if (result.error_occurred) { + if (internal::SkippedWithError == result.skipped) { printer(Out, COLOR_RED, "ERROR OCCURRED: \'%s\'", - result.error_message.c_str()); + result.skip_message.c_str()); + printer(Out, COLOR_DEFAULT, "\n"); + return; + } + if (internal::SkippedWithMessage == result.skipped) { + printer(Out, COLOR_WHITE, "SKIPPED: \'%s\'", result.skip_message.c_str()); printer(Out, COLOR_DEFAULT, "\n"); return; } @@ -166,9 +182,9 @@ void ConsoleReporter::PrintRunData(const Run& result) { printer(Out, COLOR_CYAN, "%10lld", result.iterations); } - for (auto& c : result.counters) { + for (const auto& c : result.counters) { const std::size_t cNameLen = - std::max(std::string::size_type(10), c.first.length()); + std::max(static_cast(10), c.first.length()); std::string s; const char* unit = ""; if (result.run_type == Run::RT_Aggregate && @@ -177,12 +193,13 @@ void ConsoleReporter::PrintRunData(const Run& result) { unit = "%"; } else { s = HumanReadableNumber(c.second.value, c.second.oneK); - if (c.second.flags & Counter::kIsRate) - unit = (c.second.flags & Counter::kInvert) ? "s" : "/s"; + if ((c.second.flags & Counter::kIsRate) != 0) { + unit = (c.second.flags & Counter::kInvert) != 0 ? "s" : "/s"; + } } - if (output_options_ & OO_Tabular) { - printer(Out, COLOR_DEFAULT, " %*s%s", cNameLen - strlen(unit), s.c_str(), - unit); + if ((output_options_ & OO_Tabular) != 0) { + printer(Out, COLOR_DEFAULT, " %*s%s", + static_cast(cNameLen - strlen(unit)), s.c_str(), unit); } else { printer(Out, COLOR_DEFAULT, " %s=%s%s", c.first.c_str(), s.c_str(), unit); } diff --git a/src/counter.cc b/src/counter.cc index cf5b78ee3a..4bdd5e9b59 100644 --- a/src/counter.cc +++ b/src/counter.cc @@ -17,28 +17,32 @@ namespace benchmark { namespace internal { +namespace { + double Finish(Counter const& c, IterationCount iterations, double cpu_time, double num_threads) { double v = c.value; - if (c.flags & Counter::kIsRate) { + if ((c.flags & Counter::kIsRate) != 0) { v /= cpu_time; } - if (c.flags & Counter::kAvgThreads) { + if ((c.flags & Counter::kAvgThreads) != 0) { v /= num_threads; } - if (c.flags & Counter::kIsIterationInvariant) { - v *= iterations; + if ((c.flags & Counter::kIsIterationInvariant) != 0) { + v *= static_cast(iterations); } - if (c.flags & Counter::kAvgIterations) { - v /= iterations; + if ((c.flags & Counter::kAvgIterations) != 0) { + v /= static_cast(iterations); } - if (c.flags & Counter::kInvert) { // Invert is *always* last. + if ((c.flags & Counter::kInvert) != 0) { // Invert is *always* last. v = 1.0 / v; } return v; } +} // namespace + void Finish(UserCounters* l, IterationCount iterations, double cpu_time, double num_threads) { for (auto& c : *l) { @@ -64,7 +68,9 @@ void Increment(UserCounters* l, UserCounters const& r) { } bool SameNames(UserCounters const& l, UserCounters const& r) { - if (&l == &r) return true; + if (&l == &r) { + return true; + } if (l.size() != r.size()) { return false; } diff --git a/src/counter.h b/src/counter.h index 1f5a58e31f..811a667a57 100644 --- a/src/counter.h +++ b/src/counter.h @@ -12,14 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef BENCHMARK_COUNTER_H_ -#define BENCHMARK_COUNTER_H_ +#ifndef BENCHMARK_SRC_COUNTER_H_ +#define BENCHMARK_SRC_COUNTER_H_ -#include "benchmark/benchmark.h" +#include "benchmark/counter.h" +#include "benchmark/export.h" +#include "benchmark/types.h" namespace benchmark { -// these counter-related functions are hidden to reduce API surface. namespace internal { void Finish(UserCounters* l, IterationCount iterations, double time, double num_threads); @@ -29,4 +30,4 @@ bool SameNames(UserCounters const& l, UserCounters const& r); } // end namespace benchmark -#endif // BENCHMARK_COUNTER_H_ +#endif // BENCHMARK_SRC_COUNTER_H_ diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index 1c5e9fa668..3e21d11f0a 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -12,29 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include -#include #include #include -#include #include -#include "benchmark/benchmark.h" +#include "benchmark/export.h" +#include "benchmark/reporter.h" +#include "benchmark_api_internal.h" #include "check.h" #include "complexity.h" -#include "string_util.h" -#include "timers.h" // File format reference: http://edoceo.com/utilitas/csv-file-format. namespace benchmark { namespace { -std::vector elements = { +const std::vector elements = { "name", "iterations", "real_time", "cpu_time", "time_unit", "bytes_per_second", "items_per_second", "label", "error_occurred", "error_message"}; -} // namespace std::string CsvEscape(const std::string& s) { std::string tmp; @@ -51,12 +47,15 @@ std::string CsvEscape(const std::string& s) { } return '"' + tmp + '"'; } +} // namespace +BENCHMARK_EXPORT bool CSVReporter::ReportContext(const Context& context) { PrintBasicContext(&GetErrorStream(), context); return true; } +BENCHMARK_EXPORT void CSVReporter::ReportRuns(const std::vector& reports) { std::ostream& Out = GetOutputStream(); @@ -64,8 +63,10 @@ void CSVReporter::ReportRuns(const std::vector& reports) { // save the names of all the user counters for (const auto& run : reports) { for (const auto& cnt : run.counters) { - if (cnt.first == "bytes_per_second" || cnt.first == "items_per_second") + if (cnt.first == "bytes_per_second" || + cnt.first == "items_per_second") { continue; + } user_counter_names_.insert(cnt.first); } } @@ -73,7 +74,9 @@ void CSVReporter::ReportRuns(const std::vector& reports) { // print the header for (auto B = elements.begin(); B != elements.end();) { Out << *B++; - if (B != elements.end()) Out << ","; + if (B != elements.end()) { + Out << ","; + } } for (auto B = user_counter_names_.begin(); B != user_counter_names_.end();) { @@ -86,8 +89,10 @@ void CSVReporter::ReportRuns(const std::vector& reports) { // check that all the current counters are saved in the name set for (const auto& run : reports) { for (const auto& cnt : run.counters) { - if (cnt.first == "bytes_per_second" || cnt.first == "items_per_second") + if (cnt.first == "bytes_per_second" || + cnt.first == "items_per_second") { continue; + } BM_CHECK(user_counter_names_.find(cnt.first) != user_counter_names_.end()) << "All counters must be present in each run. " @@ -103,13 +108,14 @@ void CSVReporter::ReportRuns(const std::vector& reports) { } } +BENCHMARK_EXPORT void CSVReporter::PrintRunData(const Run& run) { std::ostream& Out = GetOutputStream(); Out << CsvEscape(run.benchmark_name()) << ","; - if (run.error_occurred) { + if (run.skipped != 0u) { Out << std::string(elements.size() - 3, ','); - Out << "true,"; - Out << CsvEscape(run.error_message) << "\n"; + Out << std::boolalpha << (internal::SkippedWithError == run.skipped) << ","; + Out << CsvEscape(run.skip_message) << "\n"; return; } @@ -119,13 +125,21 @@ void CSVReporter::PrintRunData(const Run& run) { } Out << ","; - Out << run.GetAdjustedRealTime() << ","; - Out << run.GetAdjustedCPUTime() << ","; + if (run.run_type != Run::RT_Aggregate || + run.aggregate_unit == StatisticUnit::kTime) { + Out << run.GetAdjustedRealTime() << ","; + Out << run.GetAdjustedCPUTime() << ","; + } else { + assert(run.aggregate_unit == StatisticUnit::kPercentage); + Out << run.real_accumulated_time << ","; + Out << run.cpu_accumulated_time << ","; + } // Do not print timeLabel on bigO and RMS report if (run.report_big_o) { Out << GetBigOString(run.complexity); - } else if (!run.report_rms) { + } else if (!run.report_rms && + run.aggregate_unit != StatisticUnit::kPercentage) { Out << GetTimeUnitString(run.time_unit); } Out << ","; @@ -155,4 +169,14 @@ void CSVReporter::PrintRunData(const Run& run) { Out << '\n'; } +BENCHMARK_EXPORT +void CSVReporter::List( + const std::vector& benchmarks) { + std::ostream& out = GetOutputStream(); + out << "name\n"; + for (const internal::BenchmarkInstance& benchmark : benchmarks) { + out << CsvEscape(benchmark.name().str()) << "\n"; + } +} + } // end namespace benchmark diff --git a/src/cycleclock.h b/src/cycleclock.h index 04f094feaa..f383b3d6b8 100644 --- a/src/cycleclock.h +++ b/src/cycleclock.h @@ -23,7 +23,7 @@ #include -#include "benchmark/benchmark.h" +#include "benchmark/macros.h" #include "internal_macros.h" #if defined(BENCHMARK_OS_MACOSX) @@ -36,7 +36,11 @@ // declarations of some other intrinsics, breaking compilation. // Therefore, we simply declare __rdtsc ourselves. See also // http://connect.microsoft.com/VisualStudio/feedback/details/262047 -#if defined(COMPILER_MSVC) && !defined(_M_IX86) && !defined(_M_ARM64) +// +// Note that MSVC defines the x64 preprocessor macros when building +// for Arm64EC, despite it using Arm64 assembly instructions. +#if defined(COMPILER_MSVC) && !defined(_M_IX86) && !defined(_M_ARM64) && \ + !defined(_M_ARM64EC) extern "C" uint64_t __rdtsc(); #pragma intrinsic(__rdtsc) #endif @@ -69,7 +73,7 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { // frequency scaling). Also note that when the Mac sleeps, this // counter pauses; it does not continue counting, nor does it // reset to zero. - return mach_absolute_time(); + return static_cast(mach_absolute_time()); #elif defined(BENCHMARK_OS_EMSCRIPTEN) // this goes above x86-specific code because old versions of Emscripten // define __x86_64__, although they have nothing to do with it. @@ -78,10 +82,13 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { int64_t ret; __asm__ volatile("rdtsc" : "=A"(ret)); return ret; -#elif defined(__x86_64__) || defined(__amd64__) + +// Note that Clang, like MSVC, defines the x64 preprocessor macros when building +// for Arm64EC, despite it using Arm64 assembly instructions. +#elif (defined(__x86_64__) || defined(__amd64__)) && !defined(__arm64ec__) uint64_t low, high; __asm__ volatile("rdtsc" : "=a"(low), "=d"(high)); - return (high << 32) | low; + return static_cast((high << 32) | low); #elif defined(__powerpc__) || defined(__ppc__) // This returns a time-base, which is not always precisely a cycle-count. #if defined(__powerpc64__) || defined(__ppc64__) @@ -114,7 +121,7 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { // when I know it will work. Otherwise, I'll use __rdtsc and hope // the code is being compiled with a non-ancient compiler. _asm rdtsc -#elif defined(COMPILER_MSVC) && defined(_M_ARM64) +#elif defined(COMPILER_MSVC) && (defined(_M_ARM64) || defined(_M_ARM64EC)) // See // https://docs.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics // and https://reviews.llvm.org/D53115 int64_t virtual_timer_value; @@ -138,7 +145,7 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { struct timespec ts = {0, 0}; clock_gettime(CLOCK_MONOTONIC, &ts); return static_cast(ts.tv_sec) * 1000000000 + ts.tv_nsec; -#elif defined(__aarch64__) +#elif defined(__aarch64__) || defined(__arm64ec__) // System timer of ARMv8 runs at a different frequency than the CPU's. // The frequency is fixed, typically in the range 1-50MHz. It can be // read at CNTFRQ special register. We assume the OS has set up @@ -180,42 +187,71 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { #elif defined(__s390__) // Covers both s390 and s390x. // Return the CPU clock. uint64_t tsc; -#if defined(BENCHMARK_OS_ZOS) && defined(COMPILER_IBMXL) - // z/OS XL compiler HLASM syntax. +#if defined(BENCHMARK_OS_ZOS) + // z/OS HLASM syntax. asm(" stck %0" : "=m"(tsc) : : "cc"); #else + // Linux on Z syntax. asm("stck %0" : "=Q"(tsc) : : "cc"); #endif return tsc; #elif defined(__riscv) // RISC-V - // Use RDCYCLE (and RDCYCLEH on riscv32) + // Use RDTIME (and RDTIMEH on riscv32). + // RDCYCLE is a privileged instruction since Linux 6.6. #if __riscv_xlen == 32 uint32_t cycles_lo, cycles_hi0, cycles_hi1; // This asm also includes the PowerPC overflow handling strategy, as above. // Implemented in assembly because Clang insisted on branching. asm volatile( - "rdcycleh %0\n" - "rdcycle %1\n" - "rdcycleh %2\n" + "rdtimeh %0\n" + "rdtime %1\n" + "rdtimeh %2\n" "sub %0, %0, %2\n" "seqz %0, %0\n" "sub %0, zero, %0\n" "and %1, %1, %0\n" : "=r"(cycles_hi0), "=r"(cycles_lo), "=r"(cycles_hi1)); - return (static_cast(cycles_hi1) << 32) | cycles_lo; + return static_cast((static_cast(cycles_hi1) << 32) | + cycles_lo); #else uint64_t cycles; - asm volatile("rdcycle %0" : "=r"(cycles)); - return cycles; + asm volatile("rdtime %0" : "=r"(cycles)); + return static_cast(cycles); #endif #elif defined(__e2k__) || defined(__elbrus__) struct timeval tv; gettimeofday(&tv, nullptr); return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; +#elif defined(__hexagon__) + uint64_t pcycle; + asm volatile("%0 = C15:14" : "=r"(pcycle)); + return static_cast(pcycle); +#elif defined(__alpha__) + // Alpha has a cycle counter, the PCC register, but it is an unsigned 32-bit + // integer and thus wraps every ~4s, making using it for tick counts + // unreliable beyond this time range. The real-time clock is low-precision, + // roughly ~1ms, but it is the only option that can reasonable count + // indefinitely. + struct timeval tv; + gettimeofday(&tv, nullptr); + return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; +#elif defined(__hppa__) || defined(__linux__) || defined(BENCHMARK_OS_WASI) + // Fallback for all other architectures with a recent Linux kernel, e.g.: + // HP PA-RISC provides a user-readable clock counter (cr16), but + // it's not synchronized across CPUs and only 32-bit wide when programs + // are built as 32-bit binaries. + // Same for SH-4 and possibly others. + // Use clock_gettime(CLOCK_MONOTONIC, ...) instead of gettimeofday + // because is provides nanosecond resolution. + // Initialize to always return 0 if clock_gettime fails. + struct timespec ts = {0, 0}; + clock_gettime(CLOCK_MONOTONIC, &ts); + return static_cast(ts.tv_sec) * 1000000000 + ts.tv_nsec; #else -// The soft failover to a generic implementation is automatic only for ARM. -// For other platforms the developer is expected to make an attempt to create -// a fast implementation and use generic version if nothing better is available. + // The soft failover to a generic implementation is automatic only for ARM. + // For other platforms the developer is expected to make an attempt to create + // a fast implementation and use generic version if nothing better is + // available. #error You need to define CycleTimer for your OS and CPU #endif } diff --git a/src/internal_macros.h b/src/internal_macros.h index 72ba54bad2..a0bd0e1447 100644 --- a/src/internal_macros.h +++ b/src/internal_macros.h @@ -1,8 +1,6 @@ #ifndef BENCHMARK_INTERNAL_MACROS_H_ #define BENCHMARK_INTERNAL_MACROS_H_ -#include "benchmark/benchmark.h" - /* Needed to detect STL */ #include @@ -13,11 +11,7 @@ #endif #if defined(__clang__) - #if defined(__ibmxl__) - #if !defined(COMPILER_IBMXL) - #define COMPILER_IBMXL - #endif - #elif !defined(COMPILER_CLANG) + #if !defined(COMPILER_CLANG) #define COMPILER_CLANG #endif #elif defined(_MSC_VER) @@ -44,6 +38,12 @@ #define BENCHMARK_OS_CYGWIN 1 #elif defined(_WIN32) #define BENCHMARK_OS_WINDOWS 1 + // WINAPI_FAMILY_PARTITION is defined in winapifamily.h. + // We include windows.h which implicitly includes winapifamily.h for compatibility. + #ifndef NOMINMAX + #define NOMINMAX + #endif + #include #if defined(WINAPI_FAMILY_PARTITION) #if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) #define BENCHMARK_OS_WINDOWS_WIN32 1 @@ -77,6 +77,8 @@ #define BENCHMARK_OS_NACL 1 #elif defined(__EMSCRIPTEN__) #define BENCHMARK_OS_EMSCRIPTEN 1 +#elif defined(__wasi__) + #define BENCHMARK_OS_WASI 1 #elif defined(__rtems__) #define BENCHMARK_OS_RTEMS 1 #elif defined(__Fuchsia__) @@ -87,6 +89,8 @@ #define BENCHMARK_OS_QNX 1 #elif defined(__MVS__) #define BENCHMARK_OS_ZOS 1 +#elif defined(__hexagon__) +#define BENCHMARK_OS_QURT 1 #endif #if defined(__ANDROID__) && defined(__GLIBCXX__) @@ -104,6 +108,16 @@ #define BENCHMARK_MAYBE_UNUSED #endif +#if defined(__GNUC__) || defined(__clang__) +#define PRINTF_FORMAT_STRING_FUNC(format_arg, first_idx) \ + __attribute__((format(printf, format_arg, first_idx))) +#elif defined(__MINGW32__) +#define PRINTF_FORMAT_STRING_FUNC(format_arg, first_idx) \ + __attribute__((format(__MINGW_PRINTF_FORMAT, format_arg, first_idx))) +#else +#define PRINTF_FORMAT_STRING_FUNC(format_arg, first_idx) +#endif + // clang-format on #endif // BENCHMARK_INTERNAL_MACROS_H_ diff --git a/src/json_reporter.cc b/src/json_reporter.cc index e9999e18ac..37da17ba03 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -22,16 +22,16 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/export.h" +#include "benchmark/reporter.h" +#include "benchmark/types.h" +#include "benchmark_api_internal.h" #include "complexity.h" #include "string_util.h" #include "timers.h" namespace benchmark { -namespace internal { -extern std::map* global_context; -} - namespace { std::string StrEscape(const std::string& s) { @@ -85,19 +85,25 @@ std::string FormatKV(std::string const& key, bool value) { std::string FormatKV(std::string const& key, int64_t value) { std::stringstream ss; - ss << '"' << StrEscape(key) << "\": " << value; + // We really want to just dump the integer as-is, + // without the system locale interfering. + ss << '"' << StrEscape(key) << "\": " << std::to_string(value); return ss.str(); } +std::string FormatKV(std::string const& key, int value) { + return FormatKV(key, static_cast(value)); +} + std::string FormatKV(std::string const& key, double value) { std::stringstream ss; ss << '"' << StrEscape(key) << "\": "; - if (std::isnan(value)) + if (std::isnan(value)) { ss << (value < 0 ? "-" : "") << "NaN"; - else if (std::isinf(value)) + } else if (std::isinf(value)) { ss << (value < 0 ? "-" : "") << "Infinity"; - else { + } else { const auto max_digits10 = std::numeric_limits::max_digits10; const auto max_fractional_digits10 = max_digits10 - 1; @@ -126,7 +132,7 @@ bool JSONReporter::ReportContext(const Context& context) { out << indent << FormatKV("host_name", context.sys_info.name) << ",\n"; - if (Context::executable_name) { + if (Context::executable_name != nullptr) { out << indent << FormatKV("executable", Context::executable_name) << ",\n"; } @@ -140,7 +146,15 @@ bool JSONReporter::ReportContext(const Context& context) { if (CPUInfo::Scaling::UNKNOWN != info.scaling) { out << indent << FormatKV("cpu_scaling_enabled", - info.scaling == CPUInfo::Scaling::ENABLED ? true : false) + info.scaling == CPUInfo::Scaling::ENABLED) + << ",\n"; + } + + const SystemInfo& sysinfo = context.sys_info; + if (SystemInfo::ASLR::UNKNOWN != sysinfo.ASLRStatus) { + out << indent + << FormatKV("aslr_enabled", + sysinfo.ASLRStatus == SystemInfo::ASLR::ENABLED) << ",\n"; } @@ -148,7 +162,7 @@ bool JSONReporter::ReportContext(const Context& context) { indent = std::string(6, ' '); std::string cache_indent(8, ' '); for (size_t i = 0; i < info.caches.size(); ++i) { - auto& CI = info.caches[i]; + const auto& CI = info.caches[i]; out << indent << "{\n"; out << cache_indent << FormatKV("type", CI.type) << ",\n"; out << cache_indent << FormatKV("level", static_cast(CI.level)) @@ -159,7 +173,9 @@ bool JSONReporter::ReportContext(const Context& context) { << FormatKV("num_sharing", static_cast(CI.num_sharing)) << "\n"; out << indent << "}"; - if (i != info.caches.size() - 1) out << ","; + if (i != info.caches.size() - 1) { + out << ","; + } out << "\n"; } indent = std::string(4, ' '); @@ -167,19 +183,31 @@ bool JSONReporter::ReportContext(const Context& context) { out << indent << "\"load_avg\": ["; for (auto it = info.load_avg.begin(); it != info.load_avg.end();) { out << *it++; - if (it != info.load_avg.end()) out << ","; + if (it != info.load_avg.end()) { + out << ","; + } } out << "],\n"; + out << indent << FormatKV("library_version", GetBenchmarkVersion()); + out << ",\n"; + #if defined(NDEBUG) const char build_type[] = "release"; #else const char build_type[] = "debug"; #endif out << indent << FormatKV("library_build_type", build_type); + out << ",\n"; + + // NOTE: our json schema is not strictly tied to the library version! + out << indent << FormatKV("json_schema_version", 1); + + std::map* global_context = + internal::GetGlobalContext(); - if (internal::global_context != nullptr) { - for (const auto& kv : *internal::global_context) { + if (global_context != nullptr) { + for (const auto& kv : *global_context) { out << ",\n"; out << indent << FormatKV(kv.first, kv.second); } @@ -255,9 +283,12 @@ void JSONReporter::PrintRunData(Run const& run) { BENCHMARK_UNREACHABLE(); }()) << ",\n"; } - if (run.error_occurred) { - out << indent << FormatKV("error_occurred", run.error_occurred) << ",\n"; - out << indent << FormatKV("error_message", run.error_message) << ",\n"; + if (internal::SkippedWithError == run.skipped) { + out << indent << FormatKV("error_occurred", true) << ",\n"; + out << indent << FormatKV("error_message", run.skip_message) << ",\n"; + } else if (internal::SkippedWithMessage == run.skipped) { + out << indent << FormatKV("skipped", true) << ",\n"; + out << indent << FormatKV("skip_message", run.skip_message) << ",\n"; } if (!run.report_big_o && !run.report_rms) { out << indent << FormatKV("iterations", run.iterations) << ",\n"; @@ -285,19 +316,21 @@ void JSONReporter::PrintRunData(Run const& run) { out << indent << FormatKV("rms", run.GetAdjustedCPUTime()); } - for (auto& c : run.counters) { + for (const auto& c : run.counters) { out << ",\n" << indent << FormatKV(c.first, c.second); } - if (run.memory_result) { - const MemoryManager::Result memory_result = *run.memory_result; + if (run.memory_result.memory_iterations > 0) { + const auto& memory_result = run.memory_result; out << ",\n" << indent << FormatKV("allocs_per_iter", run.allocs_per_iter); out << ",\n" << indent << FormatKV("max_bytes_used", memory_result.max_bytes_used); - auto report_if_present = [&out, &indent](const char* label, int64_t val) { - if (val != MemoryManager::TombstoneValue) + auto report_if_present = [&out, &indent](const std::string& label, + int64_t val) { + if (val != MemoryManager::TombstoneValue) { out << ",\n" << indent << FormatKV(label, val); + } }; report_if_present("total_allocated_bytes", @@ -311,7 +344,24 @@ void JSONReporter::PrintRunData(Run const& run) { out << '\n'; } -const int64_t MemoryManager::TombstoneValue = - std::numeric_limits::max(); +void JSONReporter::List( + const std::vector& benchmarks) { + std::ostream& out = GetOutputStream(); + std::string inner_indent(2, ' '); + std::string indent(4, ' '); + std::string entry_indent(6, ' '); + + // Mirror the structure of the regular output so that consumers can read + // the names from the same "benchmarks" array in both modes. + out << "{\n" << inner_indent << "\"benchmarks\": ["; + bool first = true; + for (const internal::BenchmarkInstance& benchmark : benchmarks) { + out << (first ? "\n" : ",\n") << indent << "{\n"; + out << entry_indent << FormatKV("name", benchmark.name().str()) << "\n"; + out << indent << "}"; + first = false; + } + out << "\n" << inner_indent << "]\n}\n"; +} } // end namespace benchmark diff --git a/src/log.h b/src/log.h index 48c071aded..57b7bdfc45 100644 --- a/src/log.h +++ b/src/log.h @@ -4,8 +4,6 @@ #include #include -#include "benchmark/benchmark.h" - namespace benchmark { namespace internal { @@ -23,7 +21,11 @@ class LogType { private: LogType(std::ostream* out) : out_(out) {} std::ostream* out_; - BENCHMARK_DISALLOW_COPY_AND_ASSIGN(LogType); + + // NOTE: we could use BENCHMARK_DISALLOW_COPY_AND_ASSIGN but we shouldn't have + // a dependency on benchmark.h from here. + LogType(const LogType&) = delete; + LogType& operator=(const LogType&) = delete; }; template @@ -47,13 +49,13 @@ inline int& LogLevel() { } inline LogType& GetNullLogInstance() { - static LogType log(nullptr); - return log; + static LogType null_log(static_cast(nullptr)); + return null_log; } inline LogType& GetErrorLogInstance() { - static LogType log(&std::clog); - return log; + static LogType error_log(&std::clog); + return error_log; } inline LogType& GetLogInstanceForLevel(int level) { diff --git a/src/mutex.h b/src/mutex.h index bec78d9e5f..5c973e7e81 100644 --- a/src/mutex.h +++ b/src/mutex.h @@ -3,6 +3,9 @@ #include #include +#if defined(__clang__) && !defined(_WIN32) +#include +#endif #include "check.h" @@ -76,8 +79,20 @@ class CAPABILITY("mutex") Mutex { public: Mutex() {} - void lock() ACQUIRE() { mut_.lock(); } - void unlock() RELEASE() { mut_.unlock(); } + void lock() ACQUIRE() { +#if defined(__clang__) && !defined(_WIN32) + pthread_mutex_lock(mut_.native_handle()); +#else + mut_.lock(); +#endif + } + void unlock() RELEASE() { +#if defined(__clang__) && !defined(_WIN32) + pthread_mutex_unlock(mut_.native_handle()); +#else + mut_.unlock(); +#endif + } std::mutex& native_handle() { return mut_; } private: diff --git a/src/perf_counters.cc b/src/perf_counters.cc index 582475f0ba..e6f220921c 100644 --- a/src/perf_counters.cc +++ b/src/perf_counters.cc @@ -16,9 +16,15 @@ #include #include +#include #include #if defined HAVE_LIBPFM +#include +#include +#include +#include + #include "perfmon/pfmlib.h" #include "perfmon/pfmlib_perf_event.h" #endif @@ -26,103 +32,288 @@ namespace benchmark { namespace internal { -constexpr size_t PerfCounterValues::kMaxCounters; - #if defined HAVE_LIBPFM + +size_t PerfCounterValues::Read(const std::vector& leaders) { + // Create a pointer for multiple reads + const size_t bufsize = values_.size() * sizeof(values_[0]); + char* ptr = reinterpret_cast(values_.data()); + size_t size = bufsize; + for (int lead : leaders) { + auto read_bytes = ::read(lead, ptr, size); + if (read_bytes >= ssize_t(sizeof(uint64_t))) { + // Actual data bytes are all bytes minus initial padding + std::size_t data_bytes = + static_cast(read_bytes) - sizeof(uint64_t); + // This should be very cheap since it's in hot cache + std::memmove(ptr, ptr + sizeof(uint64_t), data_bytes); + // Increment our counters + ptr += data_bytes; + size -= data_bytes; + } else { + int err = errno; + GetErrorLogInstance() << "Error reading lead " << lead << " errno:" << err + << " " << ::strerror(err) << "\n"; + return 0; + } + } + return (bufsize - size) / sizeof(uint64_t); +} + const bool PerfCounters::kSupported = true; -bool PerfCounters::Initialize() { return pfm_initialize() == PFM_SUCCESS; } +// Initializes libpfm only on the first call. Returns whether that single +// initialization was successful. +bool PerfCounters::Initialize() { + // Function-scope static gets initialized only once on first call. + static const bool success = []() { + return pfm_initialize() == PFM_SUCCESS; + }(); + return success; +} + +bool PerfCounters::IsCounterSupported(const std::string& name) { + Initialize(); + perf_event_attr attr; + std::memset(&attr, 0, sizeof(attr)); + pfm_perf_encode_arg_t arg; + std::memset(&arg, 0, sizeof(arg)); + arg.attr = &attr; + const int mode = PFM_PLM3; // user mode only + int ret = pfm_get_os_event_encoding(name.c_str(), mode, PFM_OS_PERF_EVENT_EXT, + &arg); + return (ret == PFM_SUCCESS); +} + +static std::optional> QueryCPUPMUTypes() { + std::vector types; + DIR* dir = opendir("/sys/bus/event_source/devices"); + if (!dir) { + return std::nullopt; + } + while (dirent* ent = readdir(dir)) { + std::string_view name_str = ent->d_name; + auto node_path = [&](const char* node) { + return std::string("/sys/bus/event_source/devices/") + ent->d_name + "/" + + node; + }; + struct stat st; + if (name_str == "cpu" || name_str == "cpum_cf" || + stat(node_path("cpus").c_str(), &st) == 0 || errno != ENOENT) { + int type_fd = open(node_path("type").c_str(), O_RDONLY); + if (type_fd < 0) { + closedir(dir); + return std::nullopt; + } + char type_str[32] = {}; + ssize_t res = read(type_fd, type_str, sizeof(type_str) - 1); + close(type_fd); + if (res < 0) { + closedir(dir); + return std::nullopt; + } + uint64_t type; + if (sscanf(type_str, "%" PRIu64, &type) != 1) { + closedir(dir); + return std::nullopt; + } + types.push_back(type); + } + } + closedir(dir); + return types; +} + +static std::vector GetPMUTypesForEvent(const perf_event_attr& attr) { + // Replicate generic hardware events on all CPU PMUs. + if (attr.type == PERF_TYPE_HARDWARE && attr.config < PERF_COUNT_HW_MAX) { + if (auto types = QueryCPUPMUTypes()) { + return *types; + } + } + return {0}; +} PerfCounters PerfCounters::Create( const std::vector& counter_names) { - if (counter_names.empty()) { - return NoCounters(); - } - if (counter_names.size() > PerfCounterValues::kMaxCounters) { - GetErrorLogInstance() - << counter_names.size() - << " counters were requested. The minimum is 1, the maximum is " - << PerfCounterValues::kMaxCounters << "\n"; - return NoCounters(); + if (!counter_names.empty()) { + Initialize(); } - std::vector counter_ids(counter_names.size()); - const int mode = PFM_PLM3; // user mode only + // Valid counters will populate these arrays but we start empty + std::vector valid_names; + std::vector counter_ids; + std::vector leader_ids; + + // Resize to the maximum possible + valid_names.reserve(counter_names.size()); + counter_ids.reserve(counter_names.size()); + + const int kCounterMode = PFM_PLM3; // user mode only + + // Group leads will be assigned on demand. The idea is that once we cannot + // create a counter descriptor, the reason is that this group has maxed out + // so we set the group_id again to -1 and retry - giving the algorithm a + // chance to create a new group leader to hold the next set of counters. + int group_id = -1; + + // Loop through all performance counters for (size_t i = 0; i < counter_names.size(); ++i) { - const bool is_first = i == 0; - struct perf_event_attr attr {}; - attr.size = sizeof(attr); - const int group_id = !is_first ? counter_ids[0] : -1; + // we are about to push into the valid names vector + // check if we did not reach the maximum + if (valid_names.size() == PerfCounterValues::kMaxCounters) { + // Log a message if we maxed out and stop adding + GetErrorLogInstance() + << counter_names.size() << " counters were requested. The maximum is " + << PerfCounterValues::kMaxCounters << " and " << valid_names.size() + << " were already added. All remaining counters will be ignored\n"; + // stop the loop and return what we have already + break; + } + + // Check if this name is empty const auto& name = counter_names[i]; if (name.empty()) { - GetErrorLogInstance() << "A counter name was the empty string\n"; - return NoCounters(); + GetErrorLogInstance() + << "A performance counter name was the empty string\n"; + continue; } + + // Here first means first in group, ie the group leader + const bool is_first = (group_id < 0); + + // This struct will be populated by libpfm from the counter string + // and then fed into the syscall perf_event_open + struct perf_event_attr attr {}; + attr.size = sizeof(attr); + + // This is the input struct to libpfm. pfm_perf_encode_arg_t arg{}; arg.attr = &attr; - - const int pfm_get = - pfm_get_os_event_encoding(name.c_str(), mode, PFM_OS_PERF_EVENT, &arg); + const int pfm_get = pfm_get_os_event_encoding(name.c_str(), kCounterMode, + PFM_OS_PERF_EVENT, &arg); if (pfm_get != PFM_SUCCESS) { - GetErrorLogInstance() << "Unknown counter name: " << name << "\n"; - return NoCounters(); + GetErrorLogInstance() + << "Unknown performance counter name: " << name << "\n"; + continue; } - attr.disabled = is_first; - // Note: the man page for perf_event_create suggests inerit = true and + + // We then proceed to populate the remaining fields in our attribute struct + // Note: the man page for perf_event_create suggests inherit = true and // read_format = PERF_FORMAT_GROUP don't work together, but that's not the // case. + attr.disabled = is_first; attr.inherit = true; attr.pinned = is_first; attr.exclude_kernel = true; attr.exclude_user = false; attr.exclude_hv = true; - // Read all counters in one read. - attr.read_format = PERF_FORMAT_GROUP; - - int id = -1; - static constexpr size_t kNrOfSyscallRetries = 5; - // Retry syscall as it was interrupted often (b/64774091). - for (size_t num_retries = 0; num_retries < kNrOfSyscallRetries; - ++num_retries) { - id = perf_event_open(&attr, 0, -1, group_id, 0); - if (id >= 0 || errno != EINTR) { - break; + + // Read all counters in a group in one read. + attr.read_format = PERF_FORMAT_GROUP; //| PERF_FORMAT_TOTAL_TIME_ENABLED | + // PERF_FORMAT_TOTAL_TIME_RUNNING; + + uint64_t base_config = attr.config; + for (uint64_t pmu : GetPMUTypesForEvent(attr)) { + attr.config = (pmu << PERF_PMU_TYPE_SHIFT) | base_config; + int id = -1; + while (id < 0) { + static constexpr size_t kNrOfSyscallRetries = 5; + // Retry syscall as it was interrupted often (b/64774091). + for (size_t num_retries = 0; num_retries < kNrOfSyscallRetries; + ++num_retries) { + id = perf_event_open(&attr, 0, -1, group_id, 0); + if (id >= 0 || errno != EINTR) { + break; + } + } + if (id < 0) { + // If the file descriptor is negative we might have reached a limit + // in the current group. Set the group_id to -1 and retry + if (group_id >= 0) { + // Create a new group + group_id = -1; + } else { + // At this point we have already retried to set a new group id and + // failed. We then give up. + break; + } + } + } + + // We failed to get a new file descriptor. We might have reached a hard + // hardware limit that cannot be resolved even with group multiplexing + if (id < 0) { + GetErrorLogInstance() << "***WARNING** Failed to get a file descriptor " + "for performance counter " + << name << ". Ignoring\n"; + + // We give up on this counter but try to keep going + // as the others would be fine + continue; + } + if (group_id < 0) { + // This is a leader, store and assign it to the current file descriptor + leader_ids.push_back(id); + group_id = id; } + // This is a valid counter, add it to our descriptor's list + counter_ids.push_back(id); + valid_names.push_back(name); } - if (id < 0) { - GetErrorLogInstance() - << "Failed to get a file descriptor for " << name << "\n"; + } + + // Loop through all group leaders activating them + // There is another option of starting ALL counters in a process but + // that would be far reaching an intrusion. If the user is using PMCs + // by themselves then this would have a side effect on them. It is + // friendlier to loop through all groups individually. + for (int lead : leader_ids) { + if (ioctl(lead, PERF_EVENT_IOC_ENABLE) != 0) { + // This should never happen but if it does, we give up on the + // entire batch as recovery would be a mess. + GetErrorLogInstance() << "***WARNING*** Failed to start counters. " + "Clearing out all counters.\n"; + + // Close all performance counters + for (int id : counter_ids) { + ::close(id); + } + + // Return an empty object so our internal state is still good and + // the process can continue normally without impact return NoCounters(); } - - counter_ids[i] = id; - } - if (ioctl(counter_ids[0], PERF_EVENT_IOC_ENABLE) != 0) { - GetErrorLogInstance() << "Failed to start counters\n"; - return NoCounters(); } - return PerfCounters(counter_names, std::move(counter_ids)); + return PerfCounters(std::move(valid_names), std::move(counter_ids), + std::move(leader_ids)); } void PerfCounters::CloseCounters() const { if (counter_ids_.empty()) { return; } - ioctl(counter_ids_[0], PERF_EVENT_IOC_DISABLE); + for (int lead : leader_ids_) { + ioctl(lead, PERF_EVENT_IOC_DISABLE); + } for (int fd : counter_ids_) { close(fd); } } #else // defined HAVE_LIBPFM +size_t PerfCounterValues::Read(const std::vector&) { return 0; } + const bool PerfCounters::kSupported = false; bool PerfCounters::Initialize() { return false; } +bool PerfCounters::IsCounterSupported(const std::string&) { return false; } + PerfCounters PerfCounters::Create( const std::vector& counter_names) { if (!counter_names.empty()) { - GetErrorLogInstance() << "Performance counters not supported."; + GetErrorLogInstance() << "Performance counters not supported.\n"; } return NoCounters(); } @@ -130,31 +321,10 @@ PerfCounters PerfCounters::Create( void PerfCounters::CloseCounters() const {} #endif // defined HAVE_LIBPFM -Mutex PerfCountersMeasurement::mutex_; -int PerfCountersMeasurement::ref_count_ = 0; -PerfCounters PerfCountersMeasurement::counters_ = PerfCounters::NoCounters(); - PerfCountersMeasurement::PerfCountersMeasurement( const std::vector& counter_names) : start_values_(counter_names.size()), end_values_(counter_names.size()) { - MutexLock l(mutex_); - if (ref_count_ == 0) { - counters_ = PerfCounters::Create(counter_names); - } - // We chose to increment it even if `counters_` ends up invalid, - // so that we don't keep trying to create, and also since the dtor - // will decrement regardless of `counters_`'s validity - ++ref_count_; - - BM_CHECK(!counters_.IsValid() || counters_.names() == counter_names); -} - -PerfCountersMeasurement::~PerfCountersMeasurement() { - MutexLock l(mutex_); - --ref_count_; - if (ref_count_ == 0) { - counters_ = PerfCounters::NoCounters(); - } + counters_ = PerfCounters::Create(counter_names); } PerfCounters& PerfCounters::operator=(PerfCounters&& other) noexcept { @@ -162,6 +332,7 @@ PerfCounters& PerfCounters::operator=(PerfCounters&& other) noexcept { CloseCounters(); counter_ids_ = std::move(other.counter_ids_); + leader_ids_ = std::move(other.leader_ids_); counter_names_ = std::move(other.counter_names_); } return *this; diff --git a/src/perf_counters.h b/src/perf_counters.h index 680555d4b0..23cdcc378a 100644 --- a/src/perf_counters.h +++ b/src/perf_counters.h @@ -16,11 +16,15 @@ #define BENCHMARK_PERF_COUNTERS_H #include +#include #include +#include #include #include -#include "benchmark/benchmark.h" +#include "benchmark/export.h" +#include "benchmark/macros.h" +#include "benchmark/utils.h" #include "check.h" #include "log.h" #include "mutex.h" @@ -44,18 +48,21 @@ namespace internal { // The implementation ensures the storage is inlined, and allows 0-based // indexing into the counter values. // The object is used in conjunction with a PerfCounters object, by passing it -// to Snapshot(). The values are populated such that -// perfCounters->names()[i]'s value is obtained at position i (as given by -// operator[]) of this object. -class PerfCounterValues { +// to Snapshot(). The Read() method relocates individual reads, discarding +// the initial padding from each group leader in the values buffer such that +// all user accesses through the [] operator are correct. +class BENCHMARK_EXPORT PerfCounterValues { public: explicit PerfCounterValues(size_t nr_counters) : nr_counters_(nr_counters) { BM_CHECK_LE(nr_counters_, kMaxCounters); } - uint64_t operator[](size_t pos) const { return values_[kPadding + pos]; } + // We are reading correctly now so the values don't need to skip padding + uint64_t operator[](size_t pos) const { return values_[pos]; } - static constexpr size_t kMaxCounters = 3; + // Increased the maximum to 32 only since the buffer + // is std::array<> backed + static constexpr size_t kMaxCounters = 32; private: friend class PerfCounters; @@ -66,7 +73,14 @@ class PerfCounterValues { sizeof(uint64_t) * (kPadding + nr_counters_)}; } - static constexpr size_t kPadding = 1; + // This reading is complex and as the goal of this class is to + // abstract away the intrincacies of the reading process, this is + // a better place for it + size_t Read(const std::vector& leaders); + + // Move the padding to 2 due to the reading algorithm (1st padding plus a + // current read padding) + static constexpr size_t kPadding = 2; std::array values_; const size_t nr_counters_; }; @@ -79,10 +93,11 @@ class BENCHMARK_EXPORT PerfCounters final { // True iff this platform supports performance counters. static const bool kSupported; - bool IsValid() const { return !counter_names_.empty(); } + // Returns an empty object static PerfCounters NoCounters() { return PerfCounters(); } ~PerfCounters() { CloseCounters(); } + PerfCounters() = default; PerfCounters(PerfCounters&&) = default; PerfCounters(const PerfCounters&) = delete; PerfCounters& operator=(PerfCounters&&) noexcept; @@ -92,11 +107,15 @@ class BENCHMARK_EXPORT PerfCounters final { // initialization here. static bool Initialize(); + // Check if the given counter is supported, if the app wants to + // check before passing + static bool IsCounterSupported(const std::string& name); + // Return a PerfCounters object ready to read the counters with the names // specified. The values are user-mode only. The counter name format is // implementation and OS specific. - // TODO: once we move to C++-17, this should be a std::optional, and then the - // IsValid() boolean can be dropped. + // In case of failure, this method will in the worst case return an + // empty object whose state will still be valid. static PerfCounters Create(const std::vector& counter_names); // Take a snapshot of the current value of the counters into the provided @@ -105,10 +124,7 @@ class BENCHMARK_EXPORT PerfCounters final { BENCHMARK_ALWAYS_INLINE bool Snapshot(PerfCounterValues* values) const { #ifndef BENCHMARK_OS_WINDOWS assert(values != nullptr); - assert(IsValid()); - auto buffer = values->get_data_buffer(); - auto read_bytes = ::read(counter_ids_[0], buffer.first, buffer.second); - return static_cast(read_bytes) == buffer.second; + return values->Read(leader_ids_) == counter_ids_.size(); #else (void)values; return false; @@ -120,13 +136,15 @@ class BENCHMARK_EXPORT PerfCounters final { private: PerfCounters(const std::vector& counter_names, - std::vector&& counter_ids) - : counter_ids_(std::move(counter_ids)), counter_names_(counter_names) {} - PerfCounters() = default; + std::vector&& counter_ids, std::vector&& leader_ids) + : counter_ids_(std::move(counter_ids)), + leader_ids_(std::move(leader_ids)), + counter_names_(counter_names) {} void CloseCounters() const; std::vector counter_ids_; + std::vector leader_ids_; std::vector counter_names_; }; @@ -134,33 +152,25 @@ class BENCHMARK_EXPORT PerfCounters final { class BENCHMARK_EXPORT PerfCountersMeasurement final { public: PerfCountersMeasurement(const std::vector& counter_names); - ~PerfCountersMeasurement(); - - // The only way to get to `counters_` is after ctor-ing a - // `PerfCountersMeasurement`, which means that `counters_`'s state is, here, - // decided (either invalid or valid) and won't change again even if a ctor is - // concurrently running with this. This is preferring efficiency to - // maintainability, because the address of the static can be known at compile - // time. - bool IsValid() const { - MutexLock l(mutex_); - return counters_.IsValid(); - } - BENCHMARK_ALWAYS_INLINE void Start() { - assert(IsValid()); - MutexLock l(mutex_); + size_t num_counters() const { return counters_.num_counters(); } + + const std::vector& names() const { return counters_.names(); } + + BENCHMARK_ALWAYS_INLINE bool Start() { + if (num_counters() == 0) return true; // Tell the compiler to not move instructions above/below where we take // the snapshot. ClobberMemory(); valid_read_ &= counters_.Snapshot(&start_values_); ClobberMemory(); + + return valid_read_; } BENCHMARK_ALWAYS_INLINE bool Stop( std::vector>& measurements) { - assert(IsValid()); - MutexLock l(mutex_); + if (num_counters() == 0) return true; // Tell the compiler to not move instructions above/below where we take // the snapshot. ClobberMemory(); @@ -177,16 +187,12 @@ class BENCHMARK_EXPORT PerfCountersMeasurement final { } private: - static Mutex mutex_; - GUARDED_BY(mutex_) static int ref_count_; - GUARDED_BY(mutex_) static PerfCounters counters_; + PerfCounters counters_; bool valid_read_ = true; PerfCounterValues start_values_; PerfCounterValues end_values_; }; -BENCHMARK_UNUSED static bool perf_init_anchor = PerfCounters::Initialize(); - } // namespace internal } // namespace benchmark diff --git a/src/re.h b/src/re.h index 630046782d..1486dd8778 100644 --- a/src/re.h +++ b/src/re.h @@ -15,6 +15,8 @@ #ifndef BENCHMARK_RE_H_ #define BENCHMARK_RE_H_ +#include + #include "internal_macros.h" // clang-format off @@ -33,7 +35,7 @@ // Prefer C regex libraries when compiling w/o exceptions so that we can // correctly report errors. #if defined(BENCHMARK_HAS_NO_EXCEPTIONS) && \ - defined(BENCHMARK_HAVE_STD_REGEX) && \ + defined(HAVE_STD_REGEX) && \ (defined(HAVE_GNU_POSIX_REGEX) || defined(HAVE_POSIX_REGEX)) #undef HAVE_STD_REGEX #endif @@ -121,15 +123,13 @@ inline bool Regex::Init(const std::string& spec, std::string* error) { if (ec != 0) { if (error) { size_t needed = regerror(ec, &re_, nullptr, 0); - char* errbuf = new char[needed]; - regerror(ec, &re_, errbuf, needed); + std::vector errbuf(needed); + regerror(ec, &re_, errbuf.data(), needed); // regerror returns the number of bytes necessary to null terminate // the string, so we move that when assigning to error. BM_CHECK_NE(needed, 0); - error->assign(errbuf, needed - 1); - - delete[] errbuf; + error->assign(errbuf.data(), needed - 1); } return false; diff --git a/src/reporter.cc b/src/reporter.cc index 1d2df17b90..298ff1a60e 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "benchmark/reporter.h" + #include #include #include @@ -19,42 +21,48 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/sysinfo.h" +#include "benchmark_api_internal.h" #include "check.h" #include "string_util.h" #include "timers.h" namespace benchmark { -namespace internal { -extern std::map *global_context; -} BenchmarkReporter::BenchmarkReporter() : output_stream_(&std::cout), error_stream_(&std::cerr) {} BenchmarkReporter::~BenchmarkReporter() {} -void BenchmarkReporter::PrintBasicContext(std::ostream *out, - Context const &context) { +void BenchmarkReporter::PrintBasicContext(std::ostream* out, + Context const& context) { BM_CHECK(out) << "cannot be null"; - auto &Out = *out; + auto& Out = *out; +#ifndef BENCHMARK_OS_QURT + // Date/time information is not available on QuRT. + // Attempting to get it via this call cause the binary to crash. Out << LocalDateTimeString() << "\n"; +#endif - if (context.executable_name) - Out << "Running " << context.executable_name << "\n"; + if (benchmark::BenchmarkReporter::Context::executable_name != nullptr) { + Out << "Running " << benchmark::BenchmarkReporter::Context::executable_name + << "\n"; + } - const CPUInfo &info = context.cpu_info; + const CPUInfo& info = context.cpu_info; Out << "Run on (" << info.num_cpus << " X " << (info.cycles_per_second / 1000000.0) << " MHz CPU " << ((info.num_cpus > 1) ? "s" : "") << ")\n"; - if (info.caches.size() != 0) { + if (!info.caches.empty()) { Out << "CPU Caches:\n"; - for (auto &CInfo : info.caches) { + for (const auto& CInfo : info.caches) { Out << " L" << CInfo.level << " " << CInfo.type << " " << (CInfo.size / 1024) << " KiB"; - if (CInfo.num_sharing != 0) + if (CInfo.num_sharing != 0) { Out << " (x" << (info.num_cpus / CInfo.num_sharing) << ")"; + } Out << "\n"; } } @@ -62,13 +70,18 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, Out << "Load Average: "; for (auto It = info.load_avg.begin(); It != info.load_avg.end();) { Out << StrFormat("%.2f", *It++); - if (It != info.load_avg.end()) Out << ", "; + if (It != info.load_avg.end()) { + Out << ", "; + } } Out << "\n"; } - if (internal::global_context != nullptr) { - for (const auto &kv : *internal::global_context) { + std::map* global_context = + internal::GetGlobalContext(); + + if (global_context != nullptr) { + for (const auto& kv : *global_context) { Out << kv.first << ": " << kv.second << "\n"; } } @@ -79,6 +92,12 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, "overhead.\n"; } + const SystemInfo& sysinfo = context.sys_info; + if (SystemInfo::ASLR::ENABLED == sysinfo.ASLRStatus) { + Out << "***WARNING*** ASLR is enabled, the results may have unreproducible " + "noise in them.\n"; + } + #ifndef NDEBUG Out << "***WARNING*** Library was built as DEBUG. Timings may be " "affected.\n"; @@ -86,7 +105,7 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, } // No initializer because it's already initialized to NULL. -const char *BenchmarkReporter::Context::executable_name; +const char* BenchmarkReporter::Context::executable_name; BenchmarkReporter::Context::Context() : cpu_info(CPUInfo::Get()), sys_info(SystemInfo::Get()) {} @@ -101,14 +120,26 @@ std::string BenchmarkReporter::Run::benchmark_name() const { double BenchmarkReporter::Run::GetAdjustedRealTime() const { double new_time = real_accumulated_time * GetTimeUnitMultiplier(time_unit); - if (iterations != 0) new_time /= static_cast(iterations); + if (iterations != 0) { + new_time /= static_cast(iterations); + } return new_time; } double BenchmarkReporter::Run::GetAdjustedCPUTime() const { double new_time = cpu_accumulated_time * GetTimeUnitMultiplier(time_unit); - if (iterations != 0) new_time /= static_cast(iterations); + if (iterations != 0) { + new_time /= static_cast(iterations); + } return new_time; } +void BenchmarkReporter::List( + const std::vector& benchmarks) { + std::ostream& out = GetOutputStream(); + for (const internal::BenchmarkInstance& benchmark : benchmarks) { + out << benchmark.name().str() << "\n"; + } +} + } // end namespace benchmark diff --git a/src/sleep.cc b/src/sleep.cc deleted file mode 100644 index ab59000f24..0000000000 --- a/src/sleep.cc +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2015 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "sleep.h" - -#include -#include -#include - -#include "internal_macros.h" - -#ifdef BENCHMARK_OS_WINDOWS -#include -#endif - -#ifdef BENCHMARK_OS_ZOS -#include -#endif - -namespace benchmark { -#ifdef BENCHMARK_OS_WINDOWS -// Window's Sleep takes milliseconds argument. -void SleepForMilliseconds(int milliseconds) { Sleep(milliseconds); } -void SleepForSeconds(double seconds) { - SleepForMilliseconds(static_cast(kNumMillisPerSecond * seconds)); -} -#else // BENCHMARK_OS_WINDOWS -void SleepForMicroseconds(int microseconds) { -#ifdef BENCHMARK_OS_ZOS - // z/OS does not support nanosleep. Instead call sleep() and then usleep() to - // sleep for the remaining microseconds because usleep() will fail if its - // argument is greater than 1000000. - div_t sleepTime = div(microseconds, kNumMicrosPerSecond); - int seconds = sleepTime.quot; - while (seconds != 0) seconds = sleep(seconds); - while (usleep(sleepTime.rem) == -1 && errno == EINTR) - ; -#else - struct timespec sleep_time; - sleep_time.tv_sec = microseconds / kNumMicrosPerSecond; - sleep_time.tv_nsec = (microseconds % kNumMicrosPerSecond) * kNumNanosPerMicro; - while (nanosleep(&sleep_time, &sleep_time) != 0 && errno == EINTR) - ; // Ignore signals and wait for the full interval to elapse. -#endif -} - -void SleepForMilliseconds(int milliseconds) { - SleepForMicroseconds(milliseconds * kNumMicrosPerMilli); -} - -void SleepForSeconds(double seconds) { - SleepForMicroseconds(static_cast(seconds * kNumMicrosPerSecond)); -} -#endif // BENCHMARK_OS_WINDOWS -} // end namespace benchmark diff --git a/src/sleep.h b/src/sleep.h deleted file mode 100644 index f98551afe2..0000000000 --- a/src/sleep.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef BENCHMARK_SLEEP_H_ -#define BENCHMARK_SLEEP_H_ - -namespace benchmark { -const int kNumMillisPerSecond = 1000; -const int kNumMicrosPerMilli = 1000; -const int kNumMicrosPerSecond = kNumMillisPerSecond * 1000; -const int kNumNanosPerMicro = 1000; -const int kNumNanosPerSecond = kNumNanosPerMicro * kNumMicrosPerSecond; - -void SleepForMilliseconds(int milliseconds); -void SleepForSeconds(double seconds); -} // end namespace benchmark - -#endif // BENCHMARK_SLEEP_H_ diff --git a/src/statistics.cc b/src/statistics.cc index 5ba885ab00..619bb66746 100644 --- a/src/statistics.cc +++ b/src/statistics.cc @@ -21,66 +21,89 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/reporter.h" +#include "benchmark/statistics.h" +#include "benchmark/types.h" #include "check.h" namespace benchmark { -auto StatisticsSum = [](const std::vector& v) { +const auto StatisticsSum = [](const std::vector& v) { return std::accumulate(v.begin(), v.end(), 0.0); }; double StatisticsMean(const std::vector& v) { - if (v.empty()) return 0.0; - return StatisticsSum(v) * (1.0 / v.size()); + if (v.empty()) { + return 0.0; + } + return StatisticsSum(v) * (1.0 / static_cast(v.size())); } double StatisticsMedian(const std::vector& v) { - if (v.size() < 3) return StatisticsMean(v); + if (v.size() < 3) { + return StatisticsMean(v); + } std::vector copy(v); auto center = copy.begin() + v.size() / 2; std::nth_element(copy.begin(), center, copy.end()); - // did we have an odd number of samples? - // if yes, then center is the median - // it no, then we are looking for the average between center and the value - // before - if (v.size() % 2 == 1) return *center; - auto center2 = copy.begin() + v.size() / 2 - 1; - std::nth_element(copy.begin(), center2, copy.end()); + // Did we have an odd number of samples? If yes, then center is the median. + // If not, then we are looking for the average between center and the value + // before. Instead of resorting, we just look for the max value before it, + // which is not necessarily the element immediately preceding `center` Since + // `copy` is only partially sorted by `nth_element`. + if (v.size() % 2 == 1) { + return *center; + } + auto center2 = std::max_element(copy.begin(), center); return (*center + *center2) / 2.0; } // Return the sum of the squares of this sample set -auto SumSquares = [](const std::vector& v) { +const auto SumSquares = [](const std::vector& v) { return std::inner_product(v.begin(), v.end(), v.begin(), 0.0); }; -auto Sqr = [](const double dat) { return dat * dat; }; -auto Sqrt = [](const double dat) { +const auto Sqr = [](const double dat) { return dat * dat; }; +const auto Sqrt = [](const double dat) { // Avoid NaN due to imprecision in the calculations - if (dat < 0.0) return 0.0; + if (dat < 0.0) { + return 0.0; + } return std::sqrt(dat); }; double StatisticsStdDev(const std::vector& v) { const auto mean = StatisticsMean(v); - if (v.empty()) return mean; + if (v.empty()) { + return mean; + } // Sample standard deviation is undefined for n = 1 - if (v.size() == 1) return 0.0; + if (v.size() == 1) { + return 0.0; + } - const double avg_squares = SumSquares(v) * (1.0 / v.size()); - return Sqrt(v.size() / (v.size() - 1.0) * (avg_squares - Sqr(mean))); + const double avg_squares = + SumSquares(v) * (1.0 / static_cast(v.size())); + return Sqrt(static_cast(v.size()) / + (static_cast(v.size()) - 1.0) * + (avg_squares - Sqr(mean))); } double StatisticsCV(const std::vector& v) { - if (v.size() < 2) return 0.0; + if (v.size() < 2) { + return 0.0; + } const auto stddev = StatisticsStdDev(v); const auto mean = StatisticsMean(v); + if (std::fpclassify(mean) == FP_ZERO) { + return 0.0; + } + return stddev / mean; } @@ -89,11 +112,15 @@ std::vector ComputeStats( typedef BenchmarkReporter::Run Run; std::vector results; - auto error_count = - std::count_if(reports.begin(), reports.end(), - [](Run const& run) { return run.error_occurred; }); + const auto is_successful = [](Run const& run) { + return run.skipped == internal::NotSkipped; + }; + auto successful_run = + std::find_if(reports.begin(), reports.end(), is_successful); + const auto successful_count = static_cast( + std::count_if(reports.begin(), reports.end(), is_successful)); - if (reports.size() - error_count < 2) { + if (successful_count < 2) { // We don't report aggregated data if there was a single run. return results; } @@ -102,12 +129,12 @@ std::vector ComputeStats( std::vector real_accumulated_time_stat; std::vector cpu_accumulated_time_stat; - real_accumulated_time_stat.reserve(reports.size()); - cpu_accumulated_time_stat.reserve(reports.size()); + real_accumulated_time_stat.reserve(successful_count); + cpu_accumulated_time_stat.reserve(successful_count); // All repetitions should be run with the same number of iterations so we // can take this information from the first benchmark. - const IterationCount run_iterations = reports.front().iterations; + const IterationCount run_iterations = successful_run->iterations; // create stats for user counters struct CounterStat { Counter c; @@ -115,6 +142,9 @@ std::vector ComputeStats( }; std::map counter_stats; for (Run const& r : reports) { + if (!is_successful(r)) { + continue; + } for (auto const& cnt : r.counters) { auto it = counter_stats.find(cnt.first); if (it == counter_stats.end()) { @@ -122,7 +152,7 @@ std::vector ComputeStats( .emplace(cnt.first, CounterStat{cnt.second, std::vector{}}) .first; - it->second.s.reserve(reports.size()); + it->second.s.reserve(successful_count); } else { BM_CHECK_EQ(it->second.c.flags, cnt.second.flags); } @@ -131,9 +161,11 @@ std::vector ComputeStats( // Populate the accumulators. for (Run const& run : reports) { - BM_CHECK_EQ(reports[0].benchmark_name(), run.benchmark_name()); + BM_CHECK_EQ(successful_run->benchmark_name(), run.benchmark_name()); + if (!is_successful(run)) { + continue; + } BM_CHECK_EQ(run_iterations, run.iterations); - if (run.error_occurred) continue; real_accumulated_time_stat.emplace_back(run.real_accumulated_time); cpu_accumulated_time_stat.emplace_back(run.cpu_accumulated_time); // user counters @@ -145,26 +177,30 @@ std::vector ComputeStats( } // Only add label if it is same for all runs - std::string report_label = reports[0].report_label; - for (std::size_t i = 1; i < reports.size(); i++) { - if (reports[i].report_label != report_label) { + std::string report_label = successful_run->report_label; + for (const Run& run : reports) { + if (!is_successful(run)) { + continue; + } + if (run.report_label != report_label) { report_label = ""; break; } } const double iteration_rescale_factor = - double(reports.size()) / double(run_iterations); + static_cast(successful_count) / + static_cast(run_iterations); - for (const auto& Stat : *reports[0].statistics) { + for (const auto& Stat : *successful_run->statistics) { // Get the data from the accumulator to BenchmarkReporter::Run's. Run data; - data.run_name = reports[0].run_name; - data.family_index = reports[0].family_index; - data.per_family_instance_index = reports[0].per_family_instance_index; + data.run_name = successful_run->run_name; + data.family_index = successful_run->family_index; + data.per_family_instance_index = successful_run->per_family_instance_index; data.run_type = BenchmarkReporter::Run::RT_Aggregate; - data.threads = reports[0].threads; - data.repetitions = reports[0].repetitions; + data.threads = successful_run->threads; + data.repetitions = successful_run->repetitions; data.repetition_index = Run::no_repetition_index; data.aggregate_name = Stat.name_; data.aggregate_unit = Stat.unit_; @@ -175,7 +211,7 @@ std::vector ComputeStats( // Similarly, if there are N repetitions with 1 iterations each, // an aggregate will be computed over N measurements, not 1. // Thus it is best to simply use the count of separate reports. - data.iterations = reports.size(); + data.iterations = static_cast(successful_count); data.real_accumulated_time = Stat.compute_(real_accumulated_time_stat); data.cpu_accumulated_time = Stat.compute_(cpu_accumulated_time_stat); @@ -190,7 +226,7 @@ std::vector ComputeStats( data.cpu_accumulated_time *= iteration_rescale_factor; } - data.time_unit = reports[0].time_unit; + data.time_unit = successful_run->time_unit; // user counters for (auto const& kv : counter_stats) { diff --git a/src/statistics.h b/src/statistics.h index b0d2c05e72..2e56c53608 100644 --- a/src/statistics.h +++ b/src/statistics.h @@ -18,13 +18,15 @@ #include -#include "benchmark/benchmark.h" +#include "benchmark/export.h" +#include "benchmark/reporter.h" namespace benchmark { -// Return a vector containing the mean, median and standard devation information -// (and any user-specified info) for the specified list of reports. If 'reports' -// contains less than two non-errored runs an empty vector is returned +// Return a vector containing the mean, median and standard deviation +// information (and any user-specified info) for the specified list of reports. +// If 'reports' contains less than two non-errored runs an empty vector is +// returned BENCHMARK_EXPORT std::vector ComputeStats( const std::vector& reports); diff --git a/src/string_util.cc b/src/string_util.cc index 401fa13df7..9a0d54234c 100644 --- a/src/string_util.cc +++ b/src/string_util.cc @@ -11,16 +11,17 @@ #include #include "arraysize.h" +#include "benchmark/types.h" namespace benchmark { namespace { - // kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta. -const char kBigSIUnits[] = "kMGTPEZY"; +const char* const kBigSIUnits[] = {"k", "M", "G", "T", "P", "E", "Z", "Y"}; // Kibi, Mebi, Gibi, Tebi, Pebi, Exbi, Zebi, Yobi. -const char kBigIECUnits[] = "KMGTPEZY"; +const char* const kBigIECUnits[] = {"Ki", "Mi", "Gi", "Ti", + "Pi", "Ei", "Zi", "Yi"}; // milli, micro, nano, pico, femto, atto, zepto, yocto. -const char kSmallSIUnits[] = "munpfazy"; +const char* const kSmallSIUnits[] = {"m", "u", "n", "p", "f", "a", "z", "y"}; // We require that all three arrays have the same size. static_assert(arraysize(kBigSIUnits) == arraysize(kBigIECUnits), @@ -28,133 +29,135 @@ static_assert(arraysize(kBigSIUnits) == arraysize(kBigIECUnits), static_assert(arraysize(kSmallSIUnits) == arraysize(kBigSIUnits), "Small SI and Big SI unit arrays must be the same size"); -static const int64_t kUnitsSize = arraysize(kBigSIUnits); - -void ToExponentAndMantissa(double val, double thresh, int precision, - double one_k, std::string* mantissa, - int64_t* exponent) { - std::stringstream mantissa_stream; +const int64_t kUnitsSize = arraysize(kBigSIUnits); +std::pair ToExponentAndMantissa(double val, int precision, + double one_k) { + std::string mantissa; + int64_t exponent = 0; if (val < 0) { - mantissa_stream << "-"; + mantissa = "-"; val = -val; + } else { + mantissa.clear(); } // Adjust threshold so that it never excludes things which can't be rendered // in 'precision' digits. const double adjusted_threshold = - std::max(thresh, 1.0 / std::pow(10.0, precision)); - const double big_threshold = adjusted_threshold * one_k; + std::max(1.0, 1.0 / std::pow(10.0, precision)); + const double big_threshold = (adjusted_threshold * one_k) - 1; const double small_threshold = adjusted_threshold; // Values in ]simple_threshold,small_threshold[ will be printed as-is const double simple_threshold = 0.01; + auto format_mantissa = [&](double v) { mantissa += StrFormat("%g", v); }; + + // Positive powers if (val > big_threshold) { - // Positive powers double scaled = val; for (size_t i = 0; i < arraysize(kBigSIUnits); ++i) { scaled /= one_k; if (scaled <= big_threshold) { - mantissa_stream << scaled; - *exponent = i + 1; - *mantissa = mantissa_stream.str(); - return; + format_mantissa(scaled); + exponent = static_cast(i + 1); + return std::make_pair(mantissa, exponent); } } - mantissa_stream << val; - *exponent = 0; - } else if (val < small_threshold) { - // Negative powers + format_mantissa(val); + exponent = 0; + return std::make_pair(mantissa, exponent); + } + + // Negative powers + if (val < small_threshold) { if (val < simple_threshold) { double scaled = val; for (size_t i = 0; i < arraysize(kSmallSIUnits); ++i) { scaled *= one_k; if (scaled >= small_threshold) { - mantissa_stream << scaled; - *exponent = -static_cast(i + 1); - *mantissa = mantissa_stream.str(); - return; + format_mantissa(scaled); + exponent = -static_cast(i + 1); + return std::make_pair(mantissa, exponent); } } } - mantissa_stream << val; - *exponent = 0; - } else { - mantissa_stream << val; - *exponent = 0; + format_mantissa(val); + exponent = 0; + return std::make_pair(mantissa, exponent); } - *mantissa = mantissa_stream.str(); + + format_mantissa(val); + exponent = 0; + return std::make_pair(mantissa, exponent); } std::string ExponentToPrefix(int64_t exponent, bool iec) { - if (exponent == 0) return ""; + if (exponent == 0) { + return {}; + } const int64_t index = (exponent > 0 ? exponent - 1 : -exponent - 1); - if (index >= kUnitsSize) return ""; + if (index >= kUnitsSize) { + return {}; + } - const char* array = + const char* const* array = (exponent > 0 ? (iec ? kBigIECUnits : kBigSIUnits) : kSmallSIUnits); - if (iec) - return array[index] + std::string("i"); - else - return std::string(1, array[index]); -} -std::string ToBinaryStringFullySpecified(double value, double threshold, - int precision, double one_k = 1024.0) { - std::string mantissa; - int64_t exponent; - ToExponentAndMantissa(value, threshold, precision, one_k, &mantissa, - &exponent); - return mantissa + ExponentToPrefix(exponent, false); + return std::string(array[index]); } -} // end namespace - -void AppendHumanReadable(int n, std::string* str) { - std::stringstream ss; - // Round down to the nearest SI prefix. - ss << ToBinaryStringFullySpecified(n, 1.0, 0); - *str += ss.str(); -} - -std::string HumanReadableNumber(double n, double one_k) { - // 1.1 means that figures up to 1.1k should be shown with the next unit down; - // this softens edge effects. - // 1 means that we should show one decimal place of precision. - return ToBinaryStringFullySpecified(n, 1.1, 1, one_k); +std::string ToBinaryStringFullySpecified(double value, int precision, + Counter::OneK one_k) { + auto [mantissa, exponent] = ToExponentAndMantissa( + value, precision, one_k == Counter::kIs1024 ? 1024.0 : 1000.0); + return mantissa + ExponentToPrefix(exponent, one_k == Counter::kIs1024); } +PRINTF_FORMAT_STRING_FUNC(1, 0) std::string StrFormatImp(const char* msg, va_list args) { // we might need a second shot at this, so pre-emptivly make a copy va_list args_cp; va_copy(args_cp, args); - // TODO(ericwf): use std::array for first attempt to avoid one memory - // allocation guess what the size might be - std::array local_buff; - std::size_t size = local_buff.size(); + // Use std::array for first attempt to avoid one memory allocation guess what + // the size might be + std::array local_buff = {}; + // 2015-10-08: vsnprintf is used instead of snd::vsnprintf due to a limitation // in the android-ndk - auto ret = vsnprintf(local_buff.data(), size, msg, args_cp); + auto ret = vsnprintf(local_buff.data(), local_buff.size(), msg, args_cp); va_end(args_cp); // handle empty expansion - if (ret == 0) return std::string{}; - if (static_cast(ret) < size) + if (ret == 0) { + return {}; + } + if (static_cast(ret) < local_buff.size()) { return std::string(local_buff.data()); + } // we did not provide a long enough buffer on our first attempt. // add 1 to size to account for null-byte in size cast to prevent overflow - size = static_cast(ret) + 1; + std::size_t size = static_cast(ret) + 1; auto buff_ptr = std::unique_ptr(new char[size]); // 2015-10-08: vsnprintf is used instead of snd::vsnprintf due to a limitation // in the android-ndk - vsnprintf(buff_ptr.get(), size, msg, args); + va_list args_cp2; + va_copy(args_cp2, args); + vsnprintf(buff_ptr.get(), size, msg, args_cp2); + va_end(args_cp2); return std::string(buff_ptr.get()); } +} // end namespace + +std::string HumanReadableNumber(double n, Counter::OneK one_k) { + return ToBinaryStringFullySpecified(n, 1, one_k); +} + std::string StrFormat(const char* format, ...) { va_list args; va_start(args, format); @@ -164,7 +167,9 @@ std::string StrFormat(const char* format, ...) { } std::vector StrSplit(const std::string& str, char delim) { - if (str.empty()) return {}; + if (str.empty()) { + return {}; + } std::vector ret; size_t first = 0; size_t next = str.find(delim); diff --git a/src/string_util.h b/src/string_util.h index 4145861835..1a846668ba 100644 --- a/src/string_util.h +++ b/src/string_util.h @@ -4,24 +4,19 @@ #include #include #include +#include +#include "benchmark/counter.h" #include "benchmark/export.h" -#include "internal_macros.h" +#include "check.h" namespace benchmark { -void AppendHumanReadable(int n, std::string* str); - -std::string HumanReadableNumber(double n, double one_k = 1024.0); +BENCHMARK_EXPORT +std::string HumanReadableNumber(double n, Counter::OneK one_k); BENCHMARK_EXPORT -#if defined(__MINGW32__) -__attribute__((format(__MINGW_PRINTF_FORMAT, 1, 2))) -#elif defined(__GNUC__) -__attribute__((format(printf, 1, 2))) -#endif -std::string -StrFormat(const char* format, ...); +std::string StrFormat(const char* format, ...) PRINTF_FORMAT_STRING_FUNC(1, 2); inline std::ostream& StrCatImp(std::ostream& out) BENCHMARK_NOEXCEPT { return out; diff --git a/src/sysinfo.cc b/src/sysinfo.cc index 8e536905b8..a291e10e2b 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -15,6 +15,10 @@ #include "internal_macros.h" #ifdef BENCHMARK_OS_WINDOWS +#if !defined(WINVER) || WINVER < 0x0600 +#undef WINVER +#define WINVER 0x0600 +#endif // WINVER handling #include #undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA #include @@ -23,7 +27,8 @@ #include #else #include -#ifndef BENCHMARK_OS_FUCHSIA +#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) && \ + !defined(BENCHMARK_OS_WASI) #include #endif #include @@ -38,10 +43,24 @@ #endif #if defined(BENCHMARK_OS_SOLARIS) #include +#include #endif #if defined(BENCHMARK_OS_QNX) #include #endif +#if defined(BENCHMARK_OS_QURT) +#include +#endif +#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY) +#if defined(BENCHMARK_OS_FREEBSD) +#include +#endif +#include +#endif + +#if defined(BENCHMARK_OS_LINUX) +#include +#endif #include #include @@ -58,20 +77,23 @@ #include #include #include +#include #include #include +#include "benchmark/export.h" +#include "benchmark/sysinfo.h" +#include "benchmark/utils.h" #include "check.h" #include "cycleclock.h" -#include "internal_macros.h" #include "log.h" -#include "sleep.h" #include "string_util.h" +#include "timers.h" namespace benchmark { namespace { -void PrintImp(std::ostream& out) { out << std::endl; } +void PrintImp(std::ostream& out) { out << '\n'; } template void PrintImp(std::ostream& out, First&& f, Rest&&... rest) { @@ -82,6 +104,7 @@ void PrintImp(std::ostream& out, First&& f, Rest&&... rest) { template BENCHMARK_NORETURN void PrintErrorAndDie(Args&&... args) { PrintImp(std::cerr, std::forward(args)...); + std::cerr << std::flush; std::exit(EXIT_FAILURE); } @@ -107,7 +130,7 @@ struct ValueUnion { explicit ValueUnion(std::size_t buff_size) : size(sizeof(DataT) + buff_size), - buff(::new (std::malloc(size)) DataT(), &std::free) {} + buff(::new(std::malloc(size)) DataT(), &std::free) {} ValueUnion(ValueUnion&& other) = default; @@ -126,7 +149,7 @@ struct ValueUnion { } template - std::array GetAsArray() { + BENCHMARK_MAYBE_UNUSED std::array GetAsArray() { const int arr_size = sizeof(T) * N; BM_CHECK_LE(arr_size, size); std::array arr; @@ -140,16 +163,16 @@ ValueUnion GetSysctlImp(std::string const& name) { int mib[2]; mib[0] = CTL_HW; - if ((name == "hw.ncpu") || (name == "hw.cpuspeed")) { + if ((name == "hw.ncpuonline") || (name == "hw.cpuspeed")) { ValueUnion buff(sizeof(int)); - if (name == "hw.ncpu") { - mib[1] = HW_NCPU; + if (name == "hw.ncpuonline") { + mib[1] = HW_NCPUONLINE; } else { mib[1] = HW_CPUSPEED; } - if (sysctl(mib, 2, buff.data(), &buff.Size, nullptr, 0) == -1) { + if (sysctl(mib, 2, buff.data(), &buff.size, nullptr, 0) == -1) { return ValueUnion(); } return buff; @@ -187,7 +210,8 @@ bool GetSysctl(std::string const& name, Tp* out) { } template -bool GetSysctl(std::string const& name, std::array* out) { +BENCHMARK_MAYBE_UNUSED bool GetSysctl(std::string const& name, + std::array* out) { auto buff = GetSysctlImp(name); if (!buff) return false; *out = buff.GetAsArray(); @@ -199,14 +223,18 @@ template bool ReadFromFile(std::string const& fname, ArgT* arg) { *arg = ArgT(); std::ifstream f(fname.c_str()); - if (!f.is_open()) return false; + if (!f.is_open()) { + return false; + } f >> *arg; return f.good(); } CPUInfo::Scaling CpuScaling(int num_cpus) { // We don't have a valid CPU count, so don't even bother. - if (num_cpus <= 0) return CPUInfo::Scaling::UNKNOWN; + if (num_cpus <= 0) { + return CPUInfo::Scaling::UNKNOWN; + } #if defined(BENCHMARK_OS_QNX) return CPUInfo::Scaling::UNKNOWN; #elif !defined(BENCHMARK_OS_WINDOWS) @@ -217,8 +245,9 @@ CPUInfo::Scaling CpuScaling(int num_cpus) { for (int cpu = 0; cpu < num_cpus; ++cpu) { std::string governor_file = StrCat("/sys/devices/system/cpu/cpu", cpu, "/cpufreq/scaling_governor"); - if (ReadFromFile(governor_file, &res) && res != "performance") + if (ReadFromFile(governor_file, &res) && res != "performance") { return CPUInfo::Scaling::ENABLED; + } } return CPUInfo::Scaling::DISABLED; #else @@ -233,7 +262,7 @@ int CountSetBitsInCPUMap(std::string val) { CPUMask mask(benchmark::stoul(part, nullptr, 16)); return static_cast(mask.count()); }; - std::size_t pos; + std::size_t pos = 0; int total = 0; while ((pos = val.find(',')) != std::string::npos) { total += CountBits(val.substr(0, pos)); @@ -254,28 +283,35 @@ std::vector GetCacheSizesFromKVFS() { CPUInfo::CacheInfo info; std::string fpath = StrCat(dir, "index", idx++, "/"); std::ifstream f(StrCat(fpath, "size").c_str()); - if (!f.is_open()) break; + if (!f.is_open()) { + break; + } std::string suffix; f >> info.size; - if (f.fail()) + if (f.fail()) { PrintErrorAndDie("Failed while reading file '", fpath, "size'"); + } if (f.good()) { f >> suffix; - if (f.bad()) + if (f.bad()) { PrintErrorAndDie( "Invalid cache size format: failed to read size suffix"); - else if (f && suffix != "K") + } else if (f && suffix != "K") { PrintErrorAndDie("Invalid cache size format: Expected bytes ", suffix); - else if (suffix == "K") + } else if (suffix == "K") { info.size *= 1024; + } } - if (!ReadFromFile(StrCat(fpath, "type"), &info.type)) + if (!ReadFromFile(StrCat(fpath, "type"), &info.type)) { PrintErrorAndDie("Failed to read from file ", fpath, "type"); - if (!ReadFromFile(StrCat(fpath, "level"), &info.level)) + } + if (!ReadFromFile(StrCat(fpath, "level"), &info.level)) { PrintErrorAndDie("Failed to read from file ", fpath, "level"); + } std::string map_str; - if (!ReadFromFile(StrCat(fpath, "shared_cpu_map"), &map_str)) + if (!ReadFromFile(StrCat(fpath, "shared_cpu_map"), &map_str)) { PrintErrorAndDie("Failed to read from file ", fpath, "shared_cpu_map"); + } info.num_sharing = CountSetBitsInCPUMap(map_str); res.push_back(info); } @@ -319,16 +355,19 @@ std::vector GetCacheSizesWindows() { using UPtr = std::unique_ptr; GetLogicalProcessorInformation(nullptr, &buffer_size); - UPtr buff((PInfo*)malloc(buffer_size), &std::free); - if (!GetLogicalProcessorInformation(buff.get(), &buffer_size)) + UPtr buff(static_cast(std::malloc(buffer_size)), &std::free); + if (!GetLogicalProcessorInformation(buff.get(), &buffer_size)) { PrintErrorAndDie("Failed during call to GetLogicalProcessorInformation: ", GetLastError()); + } PInfo* it = buff.get(); PInfo* end = buff.get() + (buffer_size / sizeof(PInfo)); for (; it != end; ++it) { - if (it->Relationship != RelationCache) continue; + if (it->Relationship != RelationCache) { + continue; + } using BitSet = std::bitset; BitSet b(it->ProcessorMask); // To prevent duplicates, only consider caches where CPU 0 is specified @@ -337,9 +376,14 @@ std::vector GetCacheSizesWindows() { CPUInfo::CacheInfo C; C.num_sharing = static_cast(b.count()); C.level = cache.Level; - C.size = cache.Size; + C.size = static_cast(cache.Size); C.type = "Unknown"; switch (cache.Type) { +// Windows SDK version >= 10.0.26100.0 +#ifdef NTDDI_WIN11_GE + case CacheUnknown: + break; +#endif case CacheUnified: C.type = "Unified"; break; @@ -402,6 +446,9 @@ std::vector GetCacheSizes() { return GetCacheSizesWindows(); #elif defined(BENCHMARK_OS_QNX) return GetCacheSizesQNX(); +#elif defined(BENCHMARK_OS_QURT) || defined(BENCHMARK_OS_EMSCRIPTEN) || \ + defined(BENCHMARK_OS_WASI) + return std::vector(); #else return GetCacheSizesFromKVFS(); #endif @@ -417,16 +464,27 @@ std::string GetSystemName() { #ifndef UNICODE str = std::string(hostname, DWCOUNT); #else - // Using wstring_convert, Is deprecated in C++17 - using convert_type = std::codecvt_utf8; - std::wstring_convert converter; - std::wstring wStr(hostname, DWCOUNT); - str = converter.to_bytes(wStr); + // `WideCharToMultiByte` returns `0` when conversion fails. + int len = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, hostname, + DWCOUNT, NULL, 0, NULL, NULL); + str.resize(len); + WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, hostname, DWCOUNT, &str[0], + static_cast(str.size()), NULL, NULL); #endif return str; -#else // defined(BENCHMARK_OS_WINDOWS) +#elif defined(BENCHMARK_OS_QURT) + std::string str = "Hexagon DSP"; + qurt_arch_version_t arch_version_struct; + if (qurt_sysenv_get_arch_version(&arch_version_struct) == QURT_EOK) { + str += " v"; + str += std::to_string(arch_version_struct.arch_version); + } + return str; +#elif defined(BENCHMARK_OS_WASI) + return std::string("wasi"); +#else #ifndef HOST_NAME_MAX -#ifdef BENCHMARK_HAS_SYSCTL // BSD/Mac Doesnt have HOST_NAME_MAX defined +#ifdef BENCHMARK_HAS_SYSCTL // BSD/Mac doesn't have HOST_NAME_MAX defined #define HOST_NAME_MAX 64 #elif defined(BENCHMARK_OS_NACL) #define HOST_NAME_MAX 64 @@ -434,6 +492,10 @@ std::string GetSystemName() { #define HOST_NAME_MAX 154 #elif defined(BENCHMARK_OS_RTEMS) #define HOST_NAME_MAX 256 +#elif defined(BENCHMARK_OS_SOLARIS) +#define HOST_NAME_MAX MAXHOSTNAMELEN +#elif defined(BENCHMARK_OS_ZOS) +#define HOST_NAME_MAX _POSIX_HOST_NAME_MAX #else #pragma message("HOST_NAME_MAX not defined. using 64") #define HOST_NAME_MAX 64 @@ -441,85 +503,162 @@ std::string GetSystemName() { #endif // def HOST_NAME_MAX char hostname[HOST_NAME_MAX]; int retVal = gethostname(hostname, HOST_NAME_MAX); - if (retVal != 0) return std::string(""); - return std::string(hostname); + return retVal != 0 ? std::string() : std::string(hostname); #endif // Catch-all POSIX block. } -int GetNumCPUs() { -#ifdef BENCHMARK_HAS_SYSCTL - int num_cpu = -1; - if (GetSysctl("hw.ncpu", &num_cpu)) return num_cpu; - fprintf(stderr, "Err: %s\n", strerror(errno)); - std::exit(EXIT_FAILURE); -#elif defined(BENCHMARK_OS_WINDOWS) +SystemInfo::ASLR GetASLR() { +#ifdef BENCHMARK_OS_LINUX + const auto curr_personality = personality(0xffffffff); + return (curr_personality & ADDR_NO_RANDOMIZE) ? SystemInfo::ASLR::DISABLED + : SystemInfo::ASLR::ENABLED; +#else + // FIXME: support detecting ASLR on other OS. + return SystemInfo::ASLR::UNKNOWN; +#endif +} + +int GetNumCPUsImpl() { +#ifdef BENCHMARK_OS_WINDOWS SYSTEM_INFO sysinfo; // Use memset as opposed to = {} to avoid GCC missing initializer false // positives. std::memset(&sysinfo, 0, sizeof(SYSTEM_INFO)); GetSystemInfo(&sysinfo); - return sysinfo.dwNumberOfProcessors; // number of logical - // processors in the current - // group -#elif defined(BENCHMARK_OS_SOLARIS) - // Returns -1 in case of a failure. - int num_cpu = sysconf(_SC_NPROCESSORS_ONLN); - if (num_cpu < 0) { - fprintf(stderr, "sysconf(_SC_NPROCESSORS_ONLN) failed with error: %s\n", - strerror(errno)); - } - return num_cpu; + // number of logical processors in the current group + return static_cast(sysinfo.dwNumberOfProcessors); #elif defined(BENCHMARK_OS_QNX) return static_cast(_syspage_ptr->num_cpu); +#elif defined(BENCHMARK_OS_QURT) + qurt_sysenv_max_hthreads_t hardware_threads; + if (qurt_sysenv_get_max_hw_threads(&hardware_threads) != QURT_EOK) { + hardware_threads.max_hthreads = 1; + } + return static_cast(hardware_threads.max_hthreads); +#elif defined(BENCHMARK_HAS_SYSCTL) + // *BSD, macOS + int num_cpu = -1; + constexpr auto* hwncpu = +#if defined BENCHMARK_OS_MACOSX + "hw.logicalcpu"; +#elif defined(HW_NCPUONLINE) + "hw.ncpuonline"; #else - int num_cpus = 0; - int max_id = -1; - std::ifstream f("/proc/cpuinfo"); - if (!f.is_open()) { - std::cerr << "failed to open /proc/cpuinfo\n"; - return -1; + "hw.ncpu"; +#endif + if (GetSysctl(hwncpu, &num_cpu)) return num_cpu; + PrintErrorAndDie("Err: ", strerror(errno)); +#elif defined(_SC_NPROCESSORS_ONLN) + // Linux, Solaris, AIX, Haiku, WASM, etc. + // Returns -1 in case of a failure. + int num_cpu = static_cast(sysconf(_SC_NPROCESSORS_ONLN)); + if (num_cpu < 0) { + PrintErrorAndDie("sysconf(_SC_NPROCESSORS_ONLN) failed with error: ", + strerror(errno)); } - const std::string Key = "processor"; - std::string ln; - while (std::getline(f, ln)) { - if (ln.empty()) continue; - std::size_t split_idx = ln.find(':'); - std::string value; -#if defined(__s390__) - // s390 has another format in /proc/cpuinfo - // it needs to be parsed differently - if (split_idx != std::string::npos) - value = ln.substr(Key.size() + 1, split_idx - Key.size() - 1); + return num_cpu; #else - if (split_idx != std::string::npos) value = ln.substr(split_idx + 1); + // Fallback, no other API exists. + return -1; #endif - if (ln.size() >= Key.size() && ln.compare(0, Key.size(), Key) == 0) { - num_cpus++; - if (!value.empty()) { - const int cur_id = benchmark::stoi(value); - max_id = std::max(cur_id, max_id); - } - } + BENCHMARK_UNREACHABLE(); +} + +int GetNumCPUs() { + int num_cpus = GetNumCPUsImpl(); + if (num_cpus < 1) { + std::cerr << "Unable to extract number of CPUs.\n"; + // There must be at least one CPU on which we're running. + num_cpus = 1; } - if (f.bad()) { - std::cerr << "Failure reading /proc/cpuinfo\n"; - return -1; + return num_cpus; +} + +class ThreadAffinityGuard final { + public: + ThreadAffinityGuard() : reset_affinity(SetAffinity()) { + if (!reset_affinity) { + std::cerr << "***WARNING*** Failed to set thread affinity. Estimated CPU " + "frequency may be incorrect.\n"; + } } - if (!f.eof()) { - std::cerr << "Failed to read to end of /proc/cpuinfo\n"; - return -1; + + ~ThreadAffinityGuard() { + if (!reset_affinity) { + return; + } + +#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY) + int ret = pthread_setaffinity_np(self, sizeof(previous_affinity), + &previous_affinity); + if (ret == 0) { + return; + } +#elif defined(BENCHMARK_OS_WINDOWS_WIN32) + DWORD_PTR ret = SetThreadAffinityMask(self, previous_affinity); + if (ret != 0) { + return; + } +#endif // def BENCHMARK_HAS_PTHREAD_AFFINITY + PrintErrorAndDie("Failed to reset thread affinity"); } - f.close(); - if ((max_id + 1) != num_cpus) { - fprintf(stderr, - "CPU ID assignments in /proc/cpuinfo seem messed up." - " This is usually caused by a bad BIOS.\n"); + ThreadAffinityGuard(ThreadAffinityGuard&&) = delete; + ThreadAffinityGuard(const ThreadAffinityGuard&) = delete; + ThreadAffinityGuard& operator=(ThreadAffinityGuard&&) = delete; + ThreadAffinityGuard& operator=(const ThreadAffinityGuard&) = delete; + + private: + bool SetAffinity() { +#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY) + int ret = 0; + self = pthread_self(); + ret = pthread_getaffinity_np(self, sizeof(previous_affinity), + &previous_affinity); + if (ret != 0) { + return false; + } + + cpu_set_t affinity; + memcpy(&affinity, &previous_affinity, sizeof(affinity)); + + bool is_first_cpu = true; + + for (int i = 0; i < CPU_SETSIZE; ++i) { + if (CPU_ISSET(i, &affinity)) { + if (is_first_cpu) { + is_first_cpu = false; + } else { + CPU_CLR(i, &affinity); + } + } + } + + if (is_first_cpu) { + return false; + } + + ret = pthread_setaffinity_np(self, sizeof(affinity), &affinity); + return ret == 0; +#elif defined(BENCHMARK_OS_WINDOWS_WIN32) + self = GetCurrentThread(); + DWORD_PTR mask = static_cast(1) << GetCurrentProcessorNumber(); + previous_affinity = SetThreadAffinityMask(self, mask); + return previous_affinity != 0; +#else + return false; +#endif // def BENCHMARK_HAS_PTHREAD_AFFINITY } - return num_cpus; -#endif - BENCHMARK_UNREACHABLE(); -} + +#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY) + pthread_t self{}; + cpu_set_t previous_affinity{}; +#elif defined(BENCHMARK_OS_WINDOWS_WIN32) + HANDLE self; + DWORD_PTR previous_affinity; +#endif // def BENCHMARK_HAS_PTHREAD_AFFINITY + bool reset_affinity; +}; double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { // Currently, scaling is only used on linux path here, @@ -527,7 +666,7 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { (void)scaling; #if defined BENCHMARK_OS_LINUX || defined BENCHMARK_OS_CYGWIN - long freq; + long freq = 0; // If the kernel is exporting the tsc frequency use that. There are issues // where cpuinfo_max_freq cannot be relied on because the BIOS may be @@ -549,7 +688,7 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { &freq)) { // The value is in kHz (as the file name suggests). For example, on a // 2GHz warpstation, the file contains the value "2000000". - return freq * 1000.0; + return static_cast(freq) * 1000.0; } const double error_value = -1; @@ -562,7 +701,9 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { } auto StartsWithKey = [](std::string const& Value, std::string const& Key) { - if (Key.size() > Value.size()) return false; + if (Key.size() > Value.size()) { + return false; + } auto Cmp = [&](char X, char Y) { return std::tolower(X) == std::tolower(Y); }; @@ -571,22 +712,30 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { std::string ln; while (std::getline(f, ln)) { - if (ln.empty()) continue; + if (ln.empty()) { + continue; + } std::size_t split_idx = ln.find(':'); std::string value; - if (split_idx != std::string::npos) value = ln.substr(split_idx + 1); + if (split_idx != std::string::npos) { + value = ln.substr(split_idx + 1); + } // When parsing the "cpu MHz" and "bogomips" (fallback) entries, we only // accept positive values. Some environments (virtual machines) report zero, // which would cause infinite looping in WallTime_Init. if (StartsWithKey(ln, "cpu MHz")) { if (!value.empty()) { double cycles_per_second = benchmark::stod(value) * 1000000.0; - if (cycles_per_second > 0) return cycles_per_second; + if (cycles_per_second > 0) { + return cycles_per_second; + } } } else if (StartsWithKey(ln, "bogomips")) { if (!value.empty()) { bogo_clock = benchmark::stod(value) * 1000000.0; - if (bogo_clock < 0.0) bogo_clock = error_value; + if (bogo_clock < 0.0) { + bogo_clock = error_value; + } } } } @@ -602,7 +751,9 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { // If we found the bogomips clock, but nothing better, we'll use it (but // we're not happy about it); otherwise, fallback to the rough estimation // below. - if (bogo_clock >= 0.0) return bogo_clock; + if (bogo_clock >= 0.0) { + return bogo_clock; + } #elif defined BENCHMARK_HAS_SYSCTL constexpr auto* freqStr = @@ -617,9 +768,13 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { #endif unsigned long long hz = 0; #if defined BENCHMARK_OS_OPENBSD - if (GetSysctl(freqStr, &hz)) return hz * 1000000; + if (GetSysctl(freqStr, &hz)) { + return static_cast(hz * 1000000); + } #else - if (GetSysctl(freqStr, &hz)) return hz; + if (GetSysctl(freqStr, &hz)) { + return static_cast(hz); + } #endif fprintf(stderr, "Unable to determine clock rate from sysctl: %s: %s\n", freqStr, strerror(errno)); @@ -635,16 +790,18 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { SUCCEEDED( SHGetValueA(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", - "~MHz", nullptr, &data, &data_size))) - return static_cast((int64_t)data * - (int64_t)(1000 * 1000)); // was mhz + "~MHz", nullptr, &data, &data_size))) { + return static_cast(static_cast(data) * + static_cast(1000 * 1000)); // was mhz + } #elif defined(BENCHMARK_OS_SOLARIS) kstat_ctl_t* kc = kstat_open(); if (!kc) { std::cerr << "failed to open /dev/kstat\n"; return -1; } - kstat_t* ksp = kstat_lookup(kc, (char*)"cpu_info", -1, (char*)"cpu_info0"); + kstat_t* ksp = kstat_lookup(kc, const_cast("cpu_info"), -1, + const_cast("cpu_info0")); if (!ksp) { std::cerr << "failed to lookup in /dev/kstat\n"; return -1; @@ -653,8 +810,8 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { std::cerr << "failed to read from /dev/kstat\n"; return -1; } - kstat_named_t* knp = - (kstat_named_t*)kstat_data_lookup(ksp, (char*)"current_clock_Hz"); + kstat_named_t* knp = (kstat_named_t*)kstat_data_lookup( + ksp, const_cast("current_clock_Hz")); if (!knp) { std::cerr << "failed to lookup data in /dev/kstat\n"; return -1; @@ -668,28 +825,61 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { kstat_close(kc); return clock_hz; #elif defined(BENCHMARK_OS_QNX) - return static_cast((int64_t)(SYSPAGE_ENTRY(cpuinfo)->speed) * - (int64_t)(1000 * 1000)); + return static_cast( + static_cast(SYSPAGE_ENTRY(cpuinfo)->speed) * + static_cast(1000 * 1000)); +#elif defined(BENCHMARK_OS_QURT) + // QuRT doesn't provide any API to query Hexagon frequency. + return 1000000000; #endif // If we've fallen through, attempt to roughly estimate the CPU clock rate. - static constexpr int estimate_time_ms = 1000; + + // Make sure to use the same cycle counter when starting and stopping the + // cycle timer. We just pin the current thread to a cpu in the previous + // affinity set. + ThreadAffinityGuard affinity_guard; + + static constexpr double estimate_time_s = 1.0; + const double start_time = ChronoClockNow(); const auto start_ticks = cycleclock::Now(); - SleepForMilliseconds(estimate_time_ms); - return static_cast(cycleclock::Now() - start_ticks); + + // Impose load instead of calling sleep() to make sure the cycle counter + // works. + using PRNG = std::minstd_rand; + using Result = PRNG::result_type; + PRNG rng(static_cast(start_ticks)); + + Result state = 0; + + do { + static constexpr size_t batch_size = 10000; + rng.discard(batch_size); + state += rng(); + + } while (ChronoClockNow() - start_time < estimate_time_s); + + DoNotOptimize(state); + + const auto end_ticks = cycleclock::Now(); + const double end_time = ChronoClockNow(); + + return static_cast(end_ticks - start_ticks) / (end_time - start_time); + // Reset the affinity of current thread when the lifetime of affinity_guard + // ends. } std::vector GetLoadAvg() { #if (defined BENCHMARK_OS_FREEBSD || defined(BENCHMARK_OS_LINUX) || \ defined BENCHMARK_OS_MACOSX || defined BENCHMARK_OS_NETBSD || \ defined BENCHMARK_OS_OPENBSD || defined BENCHMARK_OS_DRAGONFLY) && \ - !defined(__ANDROID__) + !(defined(__ANDROID__) && __ANDROID_API__ < 29) static constexpr int kMaxSamples = 3; std::vector res(kMaxSamples, 0.0); - const int nelem = getloadavg(res.data(), kMaxSamples); + const auto nelem = getloadavg(res.data(), kMaxSamples); if (nelem < 1) { res.clear(); } else { - res.resize(nelem); + res.resize(static_cast(nelem)); } return res; #else @@ -716,5 +906,5 @@ const SystemInfo& SystemInfo::Get() { return *info; } -SystemInfo::SystemInfo() : name(GetSystemName()) {} +SystemInfo::SystemInfo() : name(GetSystemName()), ASLRStatus(GetASLR()) {} } // end namespace benchmark diff --git a/src/thread_manager.h b/src/thread_manager.h index 4680285089..80252b6117 100644 --- a/src/thread_manager.h +++ b/src/thread_manager.h @@ -3,7 +3,9 @@ #include -#include "benchmark/benchmark.h" +#include "benchmark/counter.h" +#include "benchmark/statistics.h" +#include "benchmark/types.h" #include "mutex.h" namespace benchmark { @@ -11,30 +13,15 @@ namespace internal { class ThreadManager { public: - explicit ThreadManager(int num_threads) - : alive_threads_(num_threads), start_stop_barrier_(num_threads) {} + explicit ThreadManager(int num_threads) : start_stop_barrier_(num_threads) {} Mutex& GetBenchmarkMutex() const RETURN_CAPABILITY(benchmark_mutex_) { return benchmark_mutex_; } - bool StartStopBarrier() EXCLUDES(end_cond_mutex_) { - return start_stop_barrier_.wait(); - } - - void NotifyThreadComplete() EXCLUDES(end_cond_mutex_) { - start_stop_barrier_.removeThread(); - if (--alive_threads_ == 0) { - MutexLock lock(end_cond_mutex_); - end_condition_.notify_all(); - } - } + bool StartStopBarrier() { return start_stop_barrier_.wait(); } - void WaitForAllThreads() EXCLUDES(end_cond_mutex_) { - MutexLock lock(end_cond_mutex_); - end_condition_.wait(lock.native_handle(), - [this]() { return alive_threads_ == 0; }); - } + void NotifyThreadComplete() { start_stop_barrier_.removeThread(); } struct Result { IterationCount iterations = 0; @@ -43,18 +30,15 @@ class ThreadManager { double manual_time_used = 0; int64_t complexity_n = 0; std::string report_label_; - std::string error_message_; - bool has_error_ = false; + std::string skip_message_; + internal::Skipped skipped_ = internal::NotSkipped; UserCounters counters; }; GUARDED_BY(GetBenchmarkMutex()) Result results; private: mutable Mutex benchmark_mutex_; - std::atomic alive_threads_; Barrier start_stop_barrier_; - Mutex end_cond_mutex_; - Condition end_condition_; }; } // namespace internal diff --git a/src/timers.cc b/src/timers.cc index 68612e2688..53cf4875e1 100644 --- a/src/timers.cc +++ b/src/timers.cc @@ -23,7 +23,8 @@ #include #else #include -#ifndef BENCHMARK_OS_FUCHSIA +#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) && \ + !defined(BENCHMARK_OS_WASI) #include #endif #include @@ -38,6 +39,9 @@ #include #include #endif +#if defined(BENCHMARK_OS_QURT) +#include +#endif #endif #ifdef BENCHMARK_OS_EMSCRIPTEN @@ -56,7 +60,6 @@ #include "check.h" #include "log.h" -#include "sleep.h" #include "string_util.h" namespace benchmark { @@ -65,6 +68,9 @@ namespace benchmark { #if defined(__GNUC__) #pragma GCC diagnostic ignored "-Wunused-function" #endif +#if defined(__NVCOMPILER) +#pragma diag_suppress declared_but_not_referenced +#endif namespace { #if defined(BENCHMARK_OS_WINDOWS) @@ -79,7 +85,8 @@ double MakeTime(FILETIME const& kernel_time, FILETIME const& user_time) { static_cast(user.QuadPart)) * 1e-7; } -#elif !defined(BENCHMARK_OS_FUCHSIA) +#elif !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) && \ + !defined(BENCHMARK_OS_WASI) double MakeTime(struct rusage const& ru) { return (static_cast(ru.ru_utime.tv_sec) + static_cast(ru.ru_utime.tv_usec) * 1e-6 + @@ -97,12 +104,14 @@ double MakeTime(thread_basic_info_data_t const& info) { #endif #if defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_THREAD_CPUTIME_ID) double MakeTime(struct timespec const& ts) { - return ts.tv_sec + (static_cast(ts.tv_nsec) * 1e-9); + return static_cast(ts.tv_sec) + + (static_cast(ts.tv_nsec) * 1e-9); } #endif -BENCHMARK_NORETURN static void DiagnoseAndExit(const char* msg) { - std::cerr << "ERROR: " << msg << std::endl; +BENCHMARK_NORETURN void DiagnoseAndExit(const char* msg) { + std::cerr << "ERROR: " << msg << '\n'; + std::flush(std::cerr); std::exit(EXIT_FAILURE); } @@ -119,18 +128,35 @@ double ProcessCPUUsage() { &user_time)) return MakeTime(kernel_time, user_time); DiagnoseAndExit("GetProccessTimes() failed"); +#elif defined(BENCHMARK_OS_QURT) + // Note that qurt_timer_get_ticks() is no longer documented as of SDK 5.3.0, + // and doesn't appear to work on at least some devices (eg Samsung S22), + // so let's use the actually-documented and apparently-equivalent + // qurt_sysclock_get_hw_ticks() call instead. + return static_cast( + qurt_timer_timetick_to_us(qurt_sysclock_get_hw_ticks())) * + 1.0e-6; #elif defined(BENCHMARK_OS_EMSCRIPTEN) // clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ...) returns 0 on Emscripten. // Use Emscripten-specific API. Reported CPU time would be exactly the // same as total time, but this is ok because there aren't long-latency // synchronous system calls in Emscripten. return emscripten_get_now() * 1e-3; +#elif defined(BENCHMARK_OS_WASI) + // WASI lacks CLOCK_PROCESS_CPUTIME_ID and getrusage; use monotonic clock. + struct timespec ts {}; + if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) { + return static_cast(ts.tv_sec) + + (static_cast(ts.tv_nsec) * 1e-9); + } + DiagnoseAndExit("clock_gettime(CLOCK_MONOTONIC, ...) failed"); #elif defined(CLOCK_PROCESS_CPUTIME_ID) && !defined(BENCHMARK_OS_MACOSX) // FIXME We want to use clock_gettime, but its not available in MacOS 10.11. // See https://github.com/google/benchmark/pull/292 - struct timespec spec; - if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &spec) == 0) + struct timespec spec {}; + if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &spec) == 0) { return MakeTime(spec); + } DiagnoseAndExit("clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ...) failed"); #else struct rusage ru; @@ -149,6 +175,14 @@ double ThreadCPUUsage() { GetThreadTimes(this_thread, &creation_time, &exit_time, &kernel_time, &user_time); return MakeTime(kernel_time, user_time); +#elif defined(BENCHMARK_OS_QURT) + // Note that qurt_timer_get_ticks() is no longer documented as of SDK 5.3.0, + // and doesn't appear to work on at least some devices (eg Samsung S22), + // so let's use the actually-documented and apparently-equivalent + // qurt_sysclock_get_hw_ticks() call instead. + return static_cast( + qurt_timer_timetick_to_us(qurt_sysclock_get_hw_ticks())) * + 1.0e-6; #elif defined(BENCHMARK_OS_MACOSX) // FIXME We want to use clock_gettime, but its not available in MacOS 10.11. // See https://github.com/google/benchmark/pull/292 @@ -168,13 +202,21 @@ double ThreadCPUUsage() { // RTEMS doesn't support CLOCK_THREAD_CPUTIME_ID. See // https://github.com/RTEMS/rtems/blob/master/cpukit/posix/src/clockgettime.c return ProcessCPUUsage(); +#elif defined(BENCHMARK_OS_ZOS) + // z/OS doesn't support CLOCK_THREAD_CPUTIME_ID. + return ProcessCPUUsage(); +#elif defined(BENCHMARK_OS_WASI) + // WASI doesn't support per-thread CPU timing; fall back to process time. + return ProcessCPUUsage(); #elif defined(BENCHMARK_OS_SOLARIS) struct rusage ru; if (getrusage(RUSAGE_LWP, &ru) == 0) return MakeTime(ru); DiagnoseAndExit("getrusage(RUSAGE_LWP, ...) failed"); #elif defined(CLOCK_THREAD_CPUTIME_ID) - struct timespec ts; - if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) == 0) return MakeTime(ts); + struct timespec ts {}; + if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) == 0) { + return MakeTime(ts); + } DiagnoseAndExit("clock_gettime(CLOCK_THREAD_CPUTIME_ID, ...) failed"); #else #error Per-thread timing is not available on your system. @@ -188,9 +230,9 @@ std::string LocalDateTimeString() { const std::size_t kTzOffsetLen = 6; const std::size_t kTimestampLen = 19; - std::size_t tz_len; - std::size_t timestamp_len; - long int offset_minutes; + std::size_t tz_len = 0; + std::size_t timestamp_len = 0; + long int offset_minutes = 0; char tz_offset_sign = '+'; // tz_offset is set in one of three ways: // * strftime with %z - This either returns empty or the ISO 8601 time. The @@ -210,7 +252,7 @@ std::string LocalDateTimeString() { #if defined(BENCHMARK_OS_WINDOWS) std::tm* timeinfo_p = ::localtime(&now); #else - std::tm timeinfo; + std::tm timeinfo{}; std::tm* timeinfo_p = &timeinfo; ::localtime_r(&now, &timeinfo); #endif @@ -228,9 +270,9 @@ std::string LocalDateTimeString() { tz_offset_sign = '-'; } - tz_len = + tz_len = static_cast( ::snprintf(tz_offset, sizeof(tz_offset), "%c%02li:%02li", - tz_offset_sign, offset_minutes / 100, offset_minutes % 100); + tz_offset_sign, offset_minutes / 100, offset_minutes % 100)); BM_CHECK(tz_len == kTzOffsetLen); ((void)tz_len); // Prevent unused variable warning in optimized build. } else { diff --git a/src/timers.h b/src/timers.h index 65606ccd93..690086b36c 100644 --- a/src/timers.h +++ b/src/timers.h @@ -15,6 +15,29 @@ double ChildrenCPUUsage(); // Return the CPU usage of the current thread double ThreadCPUUsage(); +#if defined(BENCHMARK_OS_QURT) + +// std::chrono::now() can return 0 on some Hexagon devices; +// this reads the value of a 56-bit, 19.2MHz hardware counter +// and converts it to seconds. Unlike std::chrono, this doesn't +// return an absolute time, but since ChronoClockNow() is only used +// to compute elapsed time, this shouldn't matter. +struct QuRTClock { + typedef uint64_t rep; + typedef std::ratio<1, 19200000> period; + typedef std::chrono::duration duration; + typedef std::chrono::time_point time_point; + static const bool is_steady = false; + + static time_point now() { + unsigned long long count; + asm volatile(" %0 = c31:30 " : "=r"(count)); + return time_point(static_cast(count)); + } +}; + +#else + #if defined(HAVE_STEADY_CLOCK) template struct ChooseSteadyClock { @@ -25,10 +48,14 @@ template <> struct ChooseSteadyClock { typedef std::chrono::steady_clock type; }; +#endif // HAVE_STEADY_CLOCK + #endif struct ChooseClockType { -#if defined(HAVE_STEADY_CLOCK) +#if defined(BENCHMARK_OS_QURT) + typedef QuRTClock type; +#elif defined(HAVE_STEADY_CLOCK) typedef ChooseSteadyClock<>::type type; #else typedef std::chrono::high_resolution_clock type; diff --git a/test/BUILD b/test/BUILD index 0a66bf3d53..d90c002f40 100644 --- a/test/BUILD +++ b/test/BUILD @@ -10,7 +10,7 @@ platform( TEST_COPTS = [ "-pedantic", "-pedantic-errors", - "-std=c++11", + "-std=c++17", "-Wall", "-Wconversion", "-Wextra", @@ -18,6 +18,14 @@ TEST_COPTS = [ # "-Wshorten-64-to-32", "-Wfloat-equal", "-fstrict-aliasing", + ## assert() are used a lot in tests upstream, which may be optimised out leading to + ## unused-variable warning. + "-Wno-unused-variable", + "-Werror=old-style-cast", +] + +TEST_MSVC_OPTS = [ + "/std:c++17", ] # Some of the issues with DoNotOptimize only occur when optimization is enabled @@ -25,14 +33,20 @@ PER_SRC_COPTS = { "donotoptimize_test.cc": ["-O3"], } -TEST_ARGS = ["--benchmark_min_time=0.01"] +TEST_ARGS = ["--benchmark_min_time=0.01s"] -PER_SRC_TEST_ARGS = ({ - "user_counters_tabular_test.cc": ["--benchmark_counters_tabular=true"], +PER_SRC_TEST_ARGS = { + "user_counters_tabular_test.cc": [ + "--benchmark_counters_tabular=true", + "--benchmark_min_time=0.2s", + ], "repetitions_test.cc": [" --benchmark_repetitions=3"], - "spec_arg_test.cc" : ["--benchmark_filter=BM_NotChosen"], - "spec_arg_verbosity_test.cc" : ["--v=42"], -}) + "spec_arg_test.cc": ["--benchmark_filter=BM_NotChosen"], + "spec_arg_verbosity_test.cc": ["--v=42"], + "complexity_test.cc": ["--benchmark_min_time=1000000x"], + "user_counters_test.cc": ["--benchmark_min_time=0.2s"], + "user_counters_threads_test.cc": ["--benchmark_min_time=0.2s"], +} cc_library( name = "output_test_helper", @@ -40,7 +54,7 @@ cc_library( srcs = ["output_test_helper.cc"], hdrs = ["output_test.h"], copts = select({ - "//:windows": [], + "//:windows": TEST_MSVC_OPTS, "//conditions:default": TEST_COPTS, }), deps = [ @@ -49,53 +63,69 @@ cc_library( ], ) +# Tests that use gtest. These rely on `gtest_main`. [ cc_test( name = test_src[:-len(".cc")], size = "small", srcs = [test_src], - args = TEST_ARGS + PER_SRC_TEST_ARGS.get(test_src, []), copts = select({ - "//:windows": [], + "//:windows": TEST_MSVC_OPTS, "//conditions:default": TEST_COPTS, - }) + PER_SRC_COPTS.get(test_src, []) , + }) + PER_SRC_COPTS.get(test_src, []), deps = [ - ":output_test_helper", "//:benchmark", "//:benchmark_internal_headers", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", - ] + ], + ) + for test_src in glob(["*_gtest.cc"]) +] + +# Tests that do not use gtest. These have their own `main` defined. +[ + cc_test( + name = test_src[:-len(".cc")], + size = "small", + srcs = [test_src], + args = TEST_ARGS + PER_SRC_TEST_ARGS.get(test_src, []), + copts = select({ + "//:windows": TEST_MSVC_OPTS, + "//conditions:default": TEST_COPTS, + }) + PER_SRC_COPTS.get(test_src, []), + deps = [ + ":output_test_helper", + "//:benchmark", + "//:benchmark_internal_headers", + ], # FIXME: Add support for assembly tests to bazel. # See Issue #556 # https://github.com/google/benchmark/issues/556 ) for test_src in glob( - ["*test.cc"], + ["*_test.cc"], exclude = [ "*_assembly_test.cc", - "cxx03_test.cc", + "cxx11_test.cc", "link_main_test.cc", ], ) ] cc_test( - name = "cxx03_test", + name = "cxx11_test", size = "small", - srcs = ["cxx03_test.cc"], - copts = TEST_COPTS + ["-std=c++03"], - deps = [ - ":output_test_helper", - "//:benchmark", - "//:benchmark_internal_headers", - "@com_google_googletest//:gtest", - "@com_google_googletest//:gtest_main", - ], + srcs = ["cxx11_test.cc"], + copts = TEST_COPTS + ["-std=c++11"], target_compatible_with = select({ "//:windows": ["@platforms//:incompatible"], "//conditions:default": [], - }) + }), + deps = [ + ":output_test_helper", + "//:benchmark_main", + ], ) cc_test( @@ -103,7 +133,7 @@ cc_test( size = "small", srcs = ["link_main_test.cc"], copts = select({ - "//:windows": [], + "//:windows": TEST_MSVC_OPTS, "//conditions:default": TEST_COPTS, }), deps = ["//:benchmark_main"], diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d528ee94a4..fe88841dd9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,10 +1,12 @@ -# Enable the tests +#Enable the tests set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) include(CheckCXXCompilerFlag) +add_cxx_compiler_flag(-Wno-unused-variable) + # NOTE: Some tests use `` to perform the test. Therefore we must # strip -DNDEBUG from the default CMake flags in DEBUG mode. string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) @@ -24,6 +26,10 @@ if( NOT uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG" ) endforeach() endif() +if (NOT BUILD_SHARED_LIBS) + add_definitions(-DBENCHMARK_STATIC_DEFINE) +endif() + check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG) set(BENCHMARK_O3_FLAG "") if (BENCHMARK_HAS_O3_FLAG) @@ -41,7 +47,10 @@ target_link_libraries(output_test_helper PRIVATE benchmark::benchmark) macro(compile_benchmark_test name) add_executable(${name} "${name}.cc") - target_link_libraries(${name} benchmark::benchmark_main ${CMAKE_THREAD_LIBS_INIT}) + target_link_libraries(${name} benchmark::benchmark ${CMAKE_THREAD_LIBS_INIT}) + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "NVHPC") + target_compile_options( ${name} PRIVATE --diag_suppress partial_override ) + endif() endmacro(compile_benchmark_test) macro(compile_benchmark_test_with_main name) @@ -55,25 +64,51 @@ macro(compile_output_test name) ${BENCHMARK_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) endmacro(compile_output_test) +macro(benchmark_add_test) + add_test(${ARGV}) + if(WIN32 AND BUILD_SHARED_LIBS) + cmake_parse_arguments(TEST "" "NAME" "" ${ARGN}) + set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() +endmacro(benchmark_add_test) + # Demonstration executable + +compile_benchmark_test_with_main(cxx11_test) +if(DEFINED MSVC) + # MSVC does not really support C++11. + set_property(TARGET cxx11_test PROPERTY CXX_STANDARD 14) +else() + set_property(TARGET cxx11_test PROPERTY CXX_STANDARD 11) +endif() +set_property(TARGET cxx11_test PROPERTY CXX_STANDARD_REQUIRED ON) +set_property(TARGET cxx11_test PROPERTY CXX_EXTENSIONS OFF) +benchmark_add_test(NAME cxx11_test COMMAND cxx11_test --benchmark_min_time=0.01s) + compile_benchmark_test(benchmark_test) -add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01) +benchmark_add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01s) compile_benchmark_test(spec_arg_test) -add_test(NAME spec_arg COMMAND spec_arg_test --benchmark_filter=BM_NotChosen) +benchmark_add_test(NAME spec_arg COMMAND spec_arg_test --benchmark_filter=BM_NotChosen) compile_benchmark_test(spec_arg_verbosity_test) -add_test(NAME spec_arg_verbosity COMMAND spec_arg_verbosity_test --v=42) +benchmark_add_test(NAME spec_arg_verbosity COMMAND spec_arg_verbosity_test --v=42) compile_benchmark_test(benchmark_setup_teardown_test) -add_test(NAME benchmark_setup_teardown COMMAND benchmark_setup_teardown_test) +benchmark_add_test(NAME benchmark_setup_teardown COMMAND benchmark_setup_teardown_test) compile_benchmark_test(filter_test) macro(add_filter_test name filter expect) - add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01 --benchmark_filter=${filter} ${expect}) - add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) + benchmark_add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01s --benchmark_filter=${filter} ${expect}) + benchmark_add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) endmacro(add_filter_test) +compile_benchmark_test(benchmark_min_time_flag_time_test) +benchmark_add_test(NAME min_time_flag_time COMMAND benchmark_min_time_flag_time_test) + +compile_benchmark_test(benchmark_min_time_flag_iters_test) +benchmark_add_test(NAME min_time_flag_iters COMMAND benchmark_min_time_flag_iters_test) + add_filter_test(filter_simple "Foo" 3) add_filter_test(filter_simple_negative "-Foo" 2) add_filter_test(filter_suffix "BM_.*" 4) @@ -94,110 +129,104 @@ add_filter_test(filter_regex_end ".*Ba$" 1) add_filter_test(filter_regex_end_negative "-.*Ba$" 4) compile_benchmark_test(options_test) -add_test(NAME options_benchmarks COMMAND options_test --benchmark_min_time=0.01) +benchmark_add_test(NAME options_benchmarks COMMAND options_test --benchmark_min_time=0.01s) compile_benchmark_test(basic_test) -add_test(NAME basic_benchmark COMMAND basic_test --benchmark_min_time=0.01) +benchmark_add_test(NAME basic_benchmark COMMAND basic_test --benchmark_min_time=0.01s) compile_output_test(repetitions_test) -add_test(NAME repetitions_benchmark COMMAND repetitions_test --benchmark_min_time=0.01 --benchmark_repetitions=3) +benchmark_add_test(NAME repetitions_benchmark COMMAND repetitions_test --benchmark_min_time=0.01s --benchmark_repetitions=3) compile_benchmark_test(diagnostics_test) -add_test(NAME diagnostics_test COMMAND diagnostics_test --benchmark_min_time=0.01) +benchmark_add_test(NAME diagnostics_test COMMAND diagnostics_test --benchmark_min_time=0.01s) compile_benchmark_test(skip_with_error_test) -add_test(NAME skip_with_error_test COMMAND skip_with_error_test --benchmark_min_time=0.01) +benchmark_add_test(NAME skip_with_error_test COMMAND skip_with_error_test --benchmark_min_time=0.01s) compile_benchmark_test(donotoptimize_test) +# Enable errors for deprecated deprecations (DoNotOptimize(Tp const& value)). +check_cxx_compiler_flag(-Werror=deprecated-declarations BENCHMARK_HAS_DEPRECATED_DECLARATIONS_FLAG) +if (BENCHMARK_HAS_DEPRECATED_DECLARATIONS_FLAG) + target_compile_options (donotoptimize_test PRIVATE "-Werror=deprecated-declarations") +endif() # Some of the issues with DoNotOptimize only occur when optimization is enabled check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG) if (BENCHMARK_HAS_O3_FLAG) set_target_properties(donotoptimize_test PROPERTIES COMPILE_FLAGS "-O3") endif() -add_test(NAME donotoptimize_test COMMAND donotoptimize_test --benchmark_min_time=0.01) +benchmark_add_test(NAME donotoptimize_test COMMAND donotoptimize_test --benchmark_min_time=0.01s) compile_benchmark_test(fixture_test) -add_test(NAME fixture_test COMMAND fixture_test --benchmark_min_time=0.01) +benchmark_add_test(NAME fixture_test COMMAND fixture_test --benchmark_min_time=0.01s) compile_benchmark_test(register_benchmark_test) -add_test(NAME register_benchmark_test COMMAND register_benchmark_test --benchmark_min_time=0.01) +benchmark_add_test(NAME register_benchmark_test COMMAND register_benchmark_test --benchmark_min_time=0.01s) compile_benchmark_test(map_test) -add_test(NAME map_test COMMAND map_test --benchmark_min_time=0.01) +benchmark_add_test(NAME map_test COMMAND map_test --benchmark_min_time=0.01s) compile_benchmark_test(multiple_ranges_test) -add_test(NAME multiple_ranges_test COMMAND multiple_ranges_test --benchmark_min_time=0.01) +benchmark_add_test(NAME multiple_ranges_test COMMAND multiple_ranges_test --benchmark_min_time=0.01s) compile_benchmark_test(args_product_test) -add_test(NAME args_product_test COMMAND args_product_test --benchmark_min_time=0.01) +benchmark_add_test(NAME args_product_test COMMAND args_product_test --benchmark_min_time=0.01s) compile_benchmark_test_with_main(link_main_test) -add_test(NAME link_main_test COMMAND link_main_test --benchmark_min_time=0.01) +benchmark_add_test(NAME link_main_test COMMAND link_main_test --benchmark_min_time=0.01s) compile_output_test(reporter_output_test) -add_test(NAME reporter_output_test COMMAND reporter_output_test --benchmark_min_time=0.01) +benchmark_add_test(NAME reporter_output_test COMMAND reporter_output_test --benchmark_min_time=0.01s) compile_output_test(templated_fixture_test) -add_test(NAME templated_fixture_test COMMAND templated_fixture_test --benchmark_min_time=0.01) +benchmark_add_test(NAME templated_fixture_test COMMAND templated_fixture_test --benchmark_min_time=0.01s) + +compile_output_test(templated_fixture_method_test) +benchmark_add_test(NAME templated_fixture_method_test COMMAND templated_fixture_method_test --benchmark_min_time=0.01s) compile_output_test(user_counters_test) -add_test(NAME user_counters_test COMMAND user_counters_test --benchmark_min_time=0.01) +benchmark_add_test(NAME user_counters_test COMMAND user_counters_test --benchmark_min_time=0.2s) + +compile_output_test(user_counters_threads_test) +benchmark_add_test(NAME user_counters_threads_test COMMAND user_counters_threads_test --benchmark_min_time=0.2s) compile_output_test(perf_counters_test) -add_test(NAME perf_counters_test COMMAND perf_counters_test --benchmark_min_time=0.01 --benchmark_perf_counters=CYCLES,BRANCHES) +benchmark_add_test(NAME perf_counters_test COMMAND perf_counters_test --benchmark_min_time=0.01s --benchmark_perf_counters=CYCLES,INSTRUCTIONS) compile_output_test(internal_threading_test) -add_test(NAME internal_threading_test COMMAND internal_threading_test --benchmark_min_time=0.01) +benchmark_add_test(NAME internal_threading_test COMMAND internal_threading_test --benchmark_min_time=0.01s) + +compile_output_test(manual_threading_test) +benchmark_add_test(NAME manual_threading_test COMMAND manual_threading_test --benchmark_min_time=0.01s) compile_output_test(report_aggregates_only_test) -add_test(NAME report_aggregates_only_test COMMAND report_aggregates_only_test --benchmark_min_time=0.01) +benchmark_add_test(NAME report_aggregates_only_test COMMAND report_aggregates_only_test --benchmark_min_time=0.01s) compile_output_test(display_aggregates_only_test) -add_test(NAME display_aggregates_only_test COMMAND display_aggregates_only_test --benchmark_min_time=0.01) +benchmark_add_test(NAME display_aggregates_only_test COMMAND display_aggregates_only_test --benchmark_min_time=0.01s) compile_output_test(user_counters_tabular_test) -add_test(NAME user_counters_tabular_test COMMAND user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01) +benchmark_add_test(NAME user_counters_tabular_test COMMAND user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.2s) compile_output_test(user_counters_thousands_test) -add_test(NAME user_counters_thousands_test COMMAND user_counters_thousands_test --benchmark_min_time=0.01) +benchmark_add_test(NAME user_counters_thousands_test COMMAND user_counters_thousands_test --benchmark_min_time=0.01s) compile_output_test(memory_manager_test) -add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01) - -# MSVC does not allow to set the language standard to C++98/03. -if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - compile_benchmark_test(cxx03_test) - set_target_properties(cxx03_test - PROPERTIES - CXX_STANDARD 98 - CXX_STANDARD_REQUIRED YES) - # libstdc++ provides different definitions within between dialects. When - # LTO is enabled and -Werror is specified GCC diagnoses this ODR violation - # causing the test to fail to compile. To prevent this we explicitly disable - # the warning. - check_cxx_compiler_flag(-Wno-odr BENCHMARK_HAS_WNO_ODR) - check_cxx_compiler_flag(-Wno-lto-type-mismatch BENCHMARK_HAS_WNO_LTO_TYPE_MISMATCH) - # Cannot set_target_properties multiple times here because the warnings will - # be overwritten on each call - set (DISABLE_LTO_WARNINGS "") - if (BENCHMARK_HAS_WNO_ODR) - set(DISABLE_LTO_WARNINGS "${DISABLE_LTO_WARNINGS} -Wno-odr") - endif() - if (BENCHMARK_HAS_WNO_LTO_TYPE_MISMATCH) - set(DISABLE_LTO_WARNINGS "${DISABLE_LTO_WARNINGS} -Wno-lto-type-mismatch") - endif() - set_target_properties(cxx03_test PROPERTIES LINK_FLAGS "${DISABLE_LTO_WARNINGS}") - add_test(NAME cxx03 COMMAND cxx03_test --benchmark_min_time=0.01) -endif() +benchmark_add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01s) + +compile_output_test(profiler_manager_test) +benchmark_add_test(NAME profiler_manager_test COMMAND profiler_manager_test --benchmark_min_time=0.01s) + +compile_benchmark_test(profiler_manager_iterations_test) +benchmark_add_test(NAME profiler_manager_iterations COMMAND profiler_manager_iterations_test) -# Attempt to work around flaky test failures when running on Appveyor servers. -if (DEFINED ENV{APPVEYOR}) - set(COMPLEXITY_MIN_TIME "0.5") -else() - set(COMPLEXITY_MIN_TIME "0.01") -endif() compile_output_test(complexity_test) -add_test(NAME complexity_benchmark COMMAND complexity_test --benchmark_min_time=${COMPLEXITY_MIN_TIME}) +benchmark_add_test(NAME complexity_benchmark COMMAND complexity_test --benchmark_min_time=1000000x) + +compile_output_test(locale_impermeability_test) +benchmark_add_test(NAME locale_impermeability_test COMMAND locale_impermeability_test) + +compile_output_test(scoped_pause_test) +benchmark_add_test(NAME scoped_pause_test COMMAND scoped_pause_test) ############################################################################### # GoogleTest Unit Tests @@ -212,7 +241,12 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) macro(add_gtest name) compile_gtest(${name}) - add_test(NAME ${name} COMMAND ${name}) + benchmark_add_test(NAME ${name} COMMAND ${name}) + if(WIN32 AND BUILD_SHARED_LIBS) + set_tests_properties(${name} PROPERTIES + ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$" + ) + endif() endmacro() add_gtest(benchmark_gtest) @@ -222,7 +256,12 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) add_gtest(statistics_gtest) add_gtest(string_util_gtest) add_gtest(perf_counters_gtest) + add_gtest(reporter_list_gtest) add_gtest(time_unit_gtest) + add_gtest(min_time_parse_gtest) + add_gtest(profiler_manager_gtest) + add_gtest(benchmark_setup_teardown_cb_types_gtest) + add_gtest(memory_results_gtest) endif(BENCHMARK_ENABLE_GTEST_TESTS) ############################################################################### @@ -264,7 +303,7 @@ if (${CMAKE_BUILD_TYPE_LOWER} MATCHES "coverage") COMMAND ${LCOV} -q -a before.lcov -a after.lcov --output-file final.lcov COMMAND ${LCOV} -q -r final.lcov "'${CMAKE_SOURCE_DIR}/test/*'" -o final.lcov COMMAND ${GENHTML} final.lcov -o lcov --demangle-cpp --sort -p "${CMAKE_BINARY_DIR}" -t benchmark - DEPENDS filter_test benchmark_test options_test basic_test fixture_test cxx03_test complexity_test + DEPENDS filter_test benchmark_test options_test basic_test fixture_test complexity_test WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMENT "Running LCOV" ) @@ -288,3 +327,16 @@ if (${CMAKE_BUILD_TYPE_LOWER} MATCHES "coverage") " --coverage flag: ${CXX_FLAG_COVERAGE_MESSAGE}") endif() endif() + +if (BENCHMARK_ENABLE_RUST_BINDINGS) + find_program(CARGO_EXECUTABLE cargo REQUIRED) + add_custom_target(build_rust_bindings ALL + COMMAND ${CARGO_EXECUTABLE} build --tests + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/bindings/rust + COMMENT "Building Rust bindings and tests" + ) + add_test(NAME rust_bindings_tests + COMMAND ${CARGO_EXECUTABLE} test + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/bindings/rust + ) +endif() diff --git a/test/args_product_test.cc b/test/args_product_test.cc index d44f391f74..5dbcc21683 100644 --- a/test/args_product_test.cc +++ b/test/args_product_test.cc @@ -3,7 +3,10 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/utils.h" class ArgsProductFixture : public ::benchmark::Fixture { public: @@ -23,7 +26,7 @@ class ArgsProductFixture : public ::benchmark::Fixture { {2, 15, 10, 9}, {4, 5, 6, 11}}) {} - void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE { + void SetUp(const ::benchmark::State& state) override { std::vector ranges = {state.range(0), state.range(1), state.range(2), state.range(3)}; @@ -34,7 +37,7 @@ class ArgsProductFixture : public ::benchmark::Fixture { // NOTE: This is not TearDown as we want to check after _all_ runs are // complete. - virtual ~ArgsProductFixture() { + ~ArgsProductFixture() override { if (actualValues != expectedValues) { std::cout << "EXPECTED\n"; for (const auto& v : expectedValues) { diff --git a/test/basic_test.cc b/test/basic_test.cc index 80389c2d9e..a20e5b4910 100644 --- a/test/basic_test.cc +++ b/test/basic_test.cc @@ -1,11 +1,18 @@ -#include "benchmark/benchmark.h" +#include "benchmark/macros.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/types.h" +#include "benchmark/utils.h" #define BASIC_BENCHMARK_TEST(x) BENCHMARK(x)->Arg(8)->Arg(512)->Arg(8192) +namespace { void BM_empty(benchmark::State& state) { for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } } BENCHMARK(BM_empty); @@ -141,11 +148,10 @@ void BM_RangedFor(benchmark::State& state) { } BENCHMARK(BM_RangedFor); -#ifdef BENCHMARK_HAS_CXX11 template void BM_OneTemplateFunc(benchmark::State& state) { auto arg = state.range(0); - T sum = 0; + BENCHMARK_UNUSED T sum = 0; for (auto _ : state) { sum += static_cast(arg); } @@ -156,8 +162,8 @@ BENCHMARK(BM_OneTemplateFunc)->Arg(1); template void BM_TwoTemplateFunc(benchmark::State& state) { auto arg = state.range(0); - A sum = 0; - B prod = 1; + BENCHMARK_UNUSED A sum = 0; + BENCHMARK_UNUSED B prod = 1; for (auto _ : state) { sum += static_cast(arg); prod *= static_cast(arg); @@ -166,8 +172,6 @@ void BM_TwoTemplateFunc(benchmark::State& state) { BENCHMARK(BM_TwoTemplateFunc)->Arg(1); BENCHMARK(BM_TwoTemplateFunc)->Arg(1); -#endif // BENCHMARK_HAS_CXX11 - // Ensure that StateIterator provides all the necessary typedefs required to // instantiate std::iterator_traits. static_assert( @@ -175,5 +179,6 @@ static_assert( benchmark::State::StateIterator>::value_type, typename benchmark::State::StateIterator::value_type>::value, ""); +} // end namespace BENCHMARK_MAIN(); diff --git a/test/benchmark_gtest.cc b/test/benchmark_gtest.cc index cfc0a0f70f..09d7c80a25 100644 --- a/test/benchmark_gtest.cc +++ b/test/benchmark_gtest.cc @@ -3,12 +3,12 @@ #include #include "../src/benchmark_register.h" +#include "benchmark/benchmark_api.h" #include "gmock/gmock.h" #include "gtest/gtest.h" namespace benchmark { namespace internal { -BENCHMARK_EXPORT extern std::map* global_context; namespace { @@ -39,7 +39,8 @@ TEST(AddRangeTest, Advanced64) { TEST(AddRangeTest, FullRange8) { std::vector dst; AddRange(&dst, int8_t{1}, std::numeric_limits::max(), 8); - EXPECT_THAT(dst, testing::ElementsAre(1, 8, 64, 127)); + EXPECT_THAT( + dst, testing::ElementsAre(int8_t{1}, int8_t{8}, int8_t{64}, int8_t{127})); } TEST(AddRangeTest, FullRange64) { @@ -129,11 +130,13 @@ TEST(AddRangeTest, FullNegativeRange64) { TEST(AddRangeTest, Simple8) { std::vector dst; - AddRange(&dst, 1, 8, 2); - EXPECT_THAT(dst, testing::ElementsAre(1, 2, 4, 8)); + AddRange(&dst, int8_t{1}, int8_t{8}, int8_t{2}); + EXPECT_THAT(dst, + testing::ElementsAre(int8_t{1}, int8_t{2}, int8_t{4}, int8_t{8})); } TEST(AddCustomContext, Simple) { + std::map*& global_context = GetGlobalContext(); EXPECT_THAT(global_context, nullptr); AddCustomContext("foo", "bar"); @@ -148,6 +151,7 @@ TEST(AddCustomContext, Simple) { } TEST(AddCustomContext, DuplicateKey) { + std::map*& global_context = GetGlobalContext(); EXPECT_THAT(global_context, nullptr); AddCustomContext("foo", "bar"); diff --git a/test/benchmark_min_time_flag_iters_test.cc b/test/benchmark_min_time_flag_iters_test.cc new file mode 100644 index 0000000000..3866ac045b --- /dev/null +++ b/test/benchmark_min_time_flag_iters_test.cc @@ -0,0 +1,71 @@ +#include +#include +#include +#include +#include + +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/reporter.h" +#include "benchmark/state.h" + +// Tests that we can specify the number of iterations with +// --benchmark_min_time=x. +namespace { + +class TestReporter : public benchmark::ConsoleReporter { + public: + bool ReportContext(const Context& context) override { + return ConsoleReporter::ReportContext(context); + }; + + void ReportRuns(const std::vector& report) override { + assert(report.size() == 1); + iter_nums_.push_back(report[0].iterations); + ConsoleReporter::ReportRuns(report); + }; + + TestReporter() {} + + ~TestReporter() override {} + + const std::vector& GetIters() const { + return iter_nums_; + } + + private: + std::vector iter_nums_; +}; + +void BM_MyBench(benchmark::State& state) { + for (auto s : state) { + } +} +} // end namespace + +BENCHMARK(BM_MyBench); + +int main(int argc, char** argv) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + + // Make a fake argv and append the new --benchmark_min_time= to it. + int fake_argc = argc + 1; + std::vector fake_argv(static_cast(fake_argc)); + for (size_t i = 0; i < static_cast(argc); ++i) { + fake_argv[i] = argv[i]; + } + fake_argv[static_cast(argc)] = "--benchmark_min_time=4x"; + + benchmark::Initialize(&fake_argc, const_cast(fake_argv.data())); + + TestReporter test_reporter; + const size_t returned_count = + benchmark::RunSpecifiedBenchmarks(&test_reporter, "BM_MyBench"); + assert(returned_count == 1); + + // Check the executed iters. + const std::vector iters = test_reporter.GetIters(); + assert(!iters.empty() && iters[0] == 4); + + return 0; +} diff --git a/test/benchmark_min_time_flag_time_test.cc b/test/benchmark_min_time_flag_time_test.cc new file mode 100644 index 0000000000..2e8f52fc67 --- /dev/null +++ b/test/benchmark_min_time_flag_time_test.cc @@ -0,0 +1,96 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/reporter.h" +#include "benchmark/state.h" + +// Tests that we can specify the min time with +// --benchmark_min_time= (no suffix needed) OR +// --benchmark_min_time=s +namespace { + +// This is from benchmark.h +typedef int64_t IterationCount; + +class TestReporter : public benchmark::ConsoleReporter { + public: + bool ReportContext(const Context& context) override { + return ConsoleReporter::ReportContext(context); + }; + + void ReportRuns(const std::vector& report) override { + assert(report.size() == 1); + ConsoleReporter::ReportRuns(report); + }; + + void ReportRunsConfig(double min_time, bool /* has_explicit_iters */, + IterationCount /* iters */) override { + min_times_.push_back(min_time); + } + + TestReporter() {} + + ~TestReporter() override {} + + const std::vector& GetMinTimes() const { return min_times_; } + + private: + std::vector min_times_; +}; + +bool AlmostEqual(double a, double b) { + return std::fabs(a - b) < std::numeric_limits::epsilon(); +} + +void DoTestHelper(int* argc, const char** argv, double expected) { + benchmark::Initialize(argc, const_cast(argv)); + + TestReporter test_reporter; + const size_t returned_count = + benchmark::RunSpecifiedBenchmarks(&test_reporter, "BM_MyBench"); + assert(returned_count == 1); + + // Check the min_time + const std::vector& min_times = test_reporter.GetMinTimes(); + assert(!min_times.empty() && AlmostEqual(min_times[0], expected)); +} + +void BM_MyBench(benchmark::State& state) { + for (auto s : state) { + } +} +BENCHMARK(BM_MyBench); + +} // end namespace + +int main(int argc, char** argv) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + + // Make a fake argv and append the new --benchmark_min_time= to it. + int fake_argc = argc + 1; + std::vector fake_argv(static_cast(fake_argc)); + + for (size_t i = 0; i < static_cast(argc); ++i) { + fake_argv[i] = argv[i]; + } + + const char* no_suffix = "--benchmark_min_time=4"; + const char* with_suffix = "--benchmark_min_time=4.0s"; + double expected = 4.0; + + fake_argv[static_cast(argc)] = no_suffix; + DoTestHelper(&fake_argc, fake_argv.data(), expected); + + fake_argv[static_cast(argc)] = with_suffix; + DoTestHelper(&fake_argc, fake_argv.data(), expected); + + return 0; +} diff --git a/test/benchmark_name_gtest.cc b/test/benchmark_name_gtest.cc index 0a6746d04d..34e97b07a3 100644 --- a/test/benchmark_name_gtest.cc +++ b/test/benchmark_name_gtest.cc @@ -1,4 +1,4 @@ -#include "benchmark/benchmark.h" +#include "benchmark/reporter.h" #include "gtest/gtest.h" namespace { diff --git a/test/benchmark_random_interleaving_gtest.cc b/test/benchmark_random_interleaving_gtest.cc index 7f2086750d..cb7f668ff5 100644 --- a/test/benchmark_random_interleaving_gtest.cc +++ b/test/benchmark_random_interleaving_gtest.cc @@ -4,7 +4,10 @@ #include "../src/commandlineflags.h" #include "../src/string_util.h" -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/reporter.h" +#include "benchmark/state.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -34,7 +37,8 @@ class EventQueue : public std::queue { } }; -EventQueue* queue = new EventQueue(); +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +EventQueue* const queue = new EventQueue(); class NullReporter : public BenchmarkReporter { public: @@ -48,7 +52,7 @@ class BenchmarkTest : public testing::Test { static void TeardownHook(int /* num_threads */) { queue->push("Teardown"); } - void Execute(const std::string& pattern) { + static void Execute(const std::string& pattern) { queue->Clear(); std::unique_ptr reporter(new NullReporter()); diff --git a/test/benchmark_setup_teardown_cb_types_gtest.cc b/test/benchmark_setup_teardown_cb_types_gtest.cc new file mode 100644 index 0000000000..716b722b68 --- /dev/null +++ b/test/benchmark_setup_teardown_cb_types_gtest.cc @@ -0,0 +1,129 @@ +#include "benchmark/benchmark_api.h" +#include "benchmark/reporter.h" +#include "benchmark/state.h" +#include "benchmark/types.h" +#include "gtest/gtest.h" + +using benchmark::Benchmark; +using benchmark::BenchmarkReporter; +using benchmark::callback_function; +using benchmark::ClearRegisteredBenchmarks; +using benchmark::RegisterBenchmark; +using benchmark::RunSpecifiedBenchmarks; +using benchmark::State; + +static int functor_called = 0; +struct Functor { + void operator()(const benchmark::State& /*unused*/) { functor_called++; } +}; + +class NullReporter : public BenchmarkReporter { + public: + bool ReportContext(const Context& /*context*/) override { return true; } + void ReportRuns(const std::vector& /* report */) override {} +}; + +class BenchmarkTest : public testing::Test { + public: + Benchmark* bm; + NullReporter null_reporter; + + int setup_calls; + int teardown_calls; + + void SetUp() override { + setup_calls = 0; + teardown_calls = 0; + functor_called = 0; + + bm = RegisterBenchmark("BM", [](State& st) { + for (auto _ : st) { + } + }); + bm->Iterations(1); + } + + void TearDown() override { ClearRegisteredBenchmarks(); } +}; + +// Test that Setup/Teardown can correctly take a lambda expressions +TEST_F(BenchmarkTest, LambdaTestCopy) { + auto setup_lambda = [this](const State&) { setup_calls++; }; + auto teardown_lambda = [this](const State&) { teardown_calls++; }; + bm->Setup(setup_lambda); + bm->Teardown(teardown_lambda); + RunSpecifiedBenchmarks(&null_reporter); + EXPECT_EQ(setup_calls, 1); + EXPECT_EQ(teardown_calls, 1); +} + +// Test that Setup/Teardown can correctly take a lambda expressions +TEST_F(BenchmarkTest, LambdaTestMove) { + auto setup_lambda = [this](const State&) { setup_calls++; }; + auto teardown_lambda = [this](const State&) { teardown_calls++; }; + bm->Setup(std::move(setup_lambda)); + bm->Teardown(std::move(teardown_lambda)); + RunSpecifiedBenchmarks(&null_reporter); + EXPECT_EQ(setup_calls, 1); + EXPECT_EQ(teardown_calls, 1); +} + +// Test that Setup/Teardown can correctly take std::function +TEST_F(BenchmarkTest, CallbackFunctionCopy) { + callback_function setup_lambda = [this](const State&) { setup_calls++; }; + callback_function teardown_lambda = [this](const State&) { + teardown_calls++; + }; + bm->Setup(setup_lambda); + bm->Teardown(teardown_lambda); + RunSpecifiedBenchmarks(&null_reporter); + EXPECT_EQ(setup_calls, 1); + EXPECT_EQ(teardown_calls, 1); +} + +// Test that Setup/Teardown can correctly take std::function +TEST_F(BenchmarkTest, CallbackFunctionMove) { + callback_function setup_lambda = [this](const State&) { setup_calls++; }; + callback_function teardown_lambda = [this](const State&) { + teardown_calls++; + }; + bm->Setup(std::move(setup_lambda)); + bm->Teardown(std::move(teardown_lambda)); + RunSpecifiedBenchmarks(&null_reporter); + EXPECT_EQ(setup_calls, 1); + EXPECT_EQ(teardown_calls, 1); +} + +// Test that Setup/Teardown can correctly take functors +TEST_F(BenchmarkTest, FunctorCopy) { + Functor func; + bm->Setup(func); + bm->Teardown(func); + RunSpecifiedBenchmarks(&null_reporter); + EXPECT_EQ(functor_called, 2); +} + +// Test that Setup/Teardown can correctly take functors +TEST_F(BenchmarkTest, FunctorMove) { + Functor func1; + Functor func2; + bm->Setup(std::move(func1)); + bm->Teardown(std::move(func2)); + RunSpecifiedBenchmarks(&null_reporter); + EXPECT_EQ(functor_called, 2); +} + +// Test that Setup/Teardown can not take nullptr +TEST_F(BenchmarkTest, NullptrTest) { +#if GTEST_HAS_DEATH_TEST + // Tests only runnable in debug mode (when BM_CHECK is enabled). +#ifndef NDEBUG +#ifndef TEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS + EXPECT_DEATH(bm->Setup(nullptr), "setup != nullptr"); + EXPECT_DEATH(bm->Teardown(nullptr), "teardown != nullptr"); +#else + GTEST_SKIP() << "Test skipped because BM_CHECK is disabled"; +#endif +#endif +#endif +} diff --git a/test/benchmark_setup_teardown_test.cc b/test/benchmark_setup_teardown_test.cc index efa34e15c1..52d2761815 100644 --- a/test/benchmark_setup_teardown_test.cc +++ b/test/benchmark_setup_teardown_test.cc @@ -2,27 +2,29 @@ #include #include #include -#include -#include #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" // Test that Setup() and Teardown() are called exactly once // for each benchmark run (single-threaded). -namespace single { +namespace { +namespace singlethreaded { static int setup_call = 0; static int teardown_call = 0; -} // namespace single +} // namespace singlethreaded +} // namespace static void DoSetup1(const benchmark::State& state) { - ++single::setup_call; + ++singlethreaded::setup_call; // Setup/Teardown should never be called with any thread_idx != 0. assert(state.thread_index() == 0); } static void DoTeardown1(const benchmark::State& state) { - ++single::teardown_call; + ++singlethreaded::teardown_call; assert(state.thread_index() == 0); } @@ -40,23 +42,24 @@ BENCHMARK(BM_with_setup) ->Teardown(DoTeardown1); // Test that Setup() and Teardown() are called once for each group of threads. +namespace { namespace concurrent { static std::atomic setup_call(0); static std::atomic teardown_call(0); static std::atomic func_call(0); } // namespace concurrent -static void DoSetup2(const benchmark::State& state) { +void DoSetup2(const benchmark::State& state) { concurrent::setup_call.fetch_add(1, std::memory_order_acquire); assert(state.thread_index() == 0); } -static void DoTeardown2(const benchmark::State& state) { +void DoTeardown2(const benchmark::State& state) { concurrent::teardown_call.fetch_add(1, std::memory_order_acquire); assert(state.thread_index() == 0); } -static void BM_concurrent(benchmark::State& state) { +void BM_concurrent(benchmark::State& state) { for (auto s : state) { } concurrent::func_call.fetch_add(1, std::memory_order_acquire); @@ -80,11 +83,11 @@ int fixture_setup = 0; class FIXTURE_BECHMARK_NAME : public ::benchmark::Fixture { public: - void SetUp(const ::benchmark::State&) BENCHMARK_OVERRIDE { + void SetUp(const ::benchmark::State& /*unused*/) override { fixture_interaction::fixture_setup++; } - ~FIXTURE_BECHMARK_NAME() {} + ~FIXTURE_BECHMARK_NAME() override {} }; BENCHMARK_F(FIXTURE_BECHMARK_NAME, BM_WithFixture)(benchmark::State& st) { @@ -92,7 +95,7 @@ BENCHMARK_F(FIXTURE_BECHMARK_NAME, BM_WithFixture)(benchmark::State& st) { } } -static void DoSetupWithFixture(const benchmark::State&) { +void DoSetupWithFixture(const benchmark::State& /*unused*/) { fixture_interaction::setup++; } @@ -110,10 +113,10 @@ namespace repetitions { int setup = 0; } -static void DoSetupWithRepetitions(const benchmark::State&) { +void DoSetupWithRepetitions(const benchmark::State& /*unused*/) { repetitions::setup++; } -static void BM_WithRep(benchmark::State& state) { +void BM_WithRep(benchmark::State& state) { for (auto _ : state) { } } @@ -126,16 +129,19 @@ BENCHMARK(BM_WithRep) ->Setup(DoSetupWithRepetitions) ->Iterations(100) ->Repetitions(4); +} // namespace int main(int argc, char** argv) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + benchmark::Initialize(&argc, argv); size_t ret = benchmark::RunSpecifiedBenchmarks("."); assert(ret > 0); // Setup/Teardown is called once for each arg group (1,3,5,7). - assert(single::setup_call == 4); - assert(single::teardown_call == 4); + assert(singlethreaded::setup_call == 4); + assert(singlethreaded::teardown_call == 4); // 3 group of threads calling this function (3,5,10). assert(concurrent::setup_call.load(std::memory_order_relaxed) == 3); @@ -145,7 +151,7 @@ int main(int argc, char** argv) { // Setup is called 4 times, once for each arg group (1,3,5,7) assert(fixture_interaction::setup == 4); - // Fixture::Setup is called everytime the bm routine is run. + // Fixture::Setup is called every time the bm routine is run. // The exact number is indeterministic, so we just assert that // it's more than setup. assert(fixture_interaction::fixture_setup > fixture_interaction::setup); diff --git a/test/benchmark_test.cc b/test/benchmark_test.cc index 47023a7eba..b98fbdfe35 100644 --- a/test/benchmark_test.cc +++ b/test/benchmark_test.cc @@ -1,23 +1,26 @@ -#include "benchmark/benchmark.h" - #include #include #include #include +#include #include #include -#include #include -#include #include +#include #include #include #include #include +#include #include #include +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/utils.h" + #if defined(__GNUC__) #define BENCHMARK_NOINLINE __attribute__((noinline)) #else @@ -42,18 +45,22 @@ double CalculatePi(int depth) { std::set ConstructRandomSet(int64_t size) { std::set s; - for (int i = 0; i < size; ++i) s.insert(s.end(), i); + for (int i = 0; i < size; ++i) { + s.insert(s.end(), i); + } return s; } +// NOLINTBEGIN(cppcoreguidelines-avoid-non-const-global-variables) std::mutex test_vector_mu; -std::vector* test_vector = nullptr; - -} // end namespace +std::optional> test_vector; +// NOLINTEND(cppcoreguidelines-avoid-non-const-global-variables) -static void BM_Factorial(benchmark::State& state) { +void BM_Factorial(benchmark::State& state) { int fac_42 = 0; - for (auto _ : state) fac_42 = Factorial(8); + for (auto _ : state) { + fac_42 = Factorial(8); + } // Prevent compiler optimizations std::stringstream ss; ss << fac_42; @@ -62,32 +69,37 @@ static void BM_Factorial(benchmark::State& state) { BENCHMARK(BM_Factorial); BENCHMARK(BM_Factorial)->UseRealTime(); -static void BM_CalculatePiRange(benchmark::State& state) { +void BM_CalculatePiRange(benchmark::State& state) { double pi = 0.0; - for (auto _ : state) pi = CalculatePi(static_cast(state.range(0))); + for (auto _ : state) { + pi = CalculatePi(static_cast(state.range(0))); + } std::stringstream ss; ss << pi; state.SetLabel(ss.str()); } BENCHMARK_RANGE(BM_CalculatePiRange, 1, 1024 * 1024); -static void BM_CalculatePi(benchmark::State& state) { +void BM_CalculatePi(benchmark::State& state) { static const int depth = 1024; for (auto _ : state) { - benchmark::DoNotOptimize(CalculatePi(static_cast(depth))); + double pi = CalculatePi(static_cast(depth)); + benchmark::DoNotOptimize(pi); } } BENCHMARK(BM_CalculatePi)->Threads(8); BENCHMARK(BM_CalculatePi)->ThreadRange(1, 32); BENCHMARK(BM_CalculatePi)->ThreadPerCpu(); -static void BM_SetInsert(benchmark::State& state) { +void BM_SetInsert(benchmark::State& state) { std::set data; for (auto _ : state) { state.PauseTiming(); data = ConstructRandomSet(state.range(0)); state.ResumeTiming(); - for (int j = 0; j < state.range(1); ++j) data.insert(rand()); + for (int j = 0; j < state.range(1); ++j) { + data.insert(rand()); + } } state.SetItemsProcessed(state.iterations() * state.range(1)); state.SetBytesProcessed(state.iterations() * state.range(1) * @@ -101,11 +113,13 @@ BENCHMARK(BM_SetInsert)->Ranges({{1 << 10, 8 << 10}, {128, 512}}); template -static void BM_Sequential(benchmark::State& state) { +void BM_Sequential(benchmark::State& state) { ValueType v = 42; for (auto _ : state) { Container c; - for (int64_t i = state.range(0); --i;) c.push_back(v); + for (int64_t i = state.range(0); --i;) { + c.push_back(v); + } } const int64_t items_processed = state.iterations() * state.range(0); state.SetItemsProcessed(items_processed); @@ -115,55 +129,58 @@ BENCHMARK_TEMPLATE2(BM_Sequential, std::vector, int) ->Range(1 << 0, 1 << 10); BENCHMARK_TEMPLATE(BM_Sequential, std::list)->Range(1 << 0, 1 << 10); // Test the variadic version of BENCHMARK_TEMPLATE in C++11 and beyond. -#ifdef BENCHMARK_HAS_CXX11 BENCHMARK_TEMPLATE(BM_Sequential, std::vector, int)->Arg(512); -#endif -static void BM_StringCompare(benchmark::State& state) { +void BM_StringCompare(benchmark::State& state) { size_t len = static_cast(state.range(0)); std::string s1(len, '-'); std::string s2(len, '-'); - for (auto _ : state) benchmark::DoNotOptimize(s1.compare(s2)); + for (auto _ : state) { + auto comp = s1.compare(s2); + benchmark::DoNotOptimize(comp); + } } BENCHMARK(BM_StringCompare)->Range(1, 1 << 20); -static void BM_SetupTeardown(benchmark::State& state) { +void BM_SetupTeardown(benchmark::State& state) { if (state.thread_index() == 0) { // No need to lock test_vector_mu here as this is running single-threaded. - test_vector = new std::vector(); + test_vector = std::vector(); } int i = 0; for (auto _ : state) { std::lock_guard l(test_vector_mu); - if (i % 2 == 0) + if (i % 2 == 0) { test_vector->push_back(i); - else + } else { test_vector->pop_back(); + } ++i; } if (state.thread_index() == 0) { - delete test_vector; + test_vector.reset(); } } BENCHMARK(BM_SetupTeardown)->ThreadPerCpu(); -static void BM_LongTest(benchmark::State& state) { +void BM_LongTest(benchmark::State& state) { double tracker = 0.0; for (auto _ : state) { - for (int i = 0; i < state.range(0); ++i) + for (int i = 0; i < state.range(0); ++i) { benchmark::DoNotOptimize(tracker += i); + } } } BENCHMARK(BM_LongTest)->Range(1 << 16, 1 << 28); -static void BM_ParallelMemset(benchmark::State& state) { +void BM_ParallelMemset(benchmark::State& state) { int64_t size = state.range(0) / static_cast(sizeof(int)); int thread_size = static_cast(size) / state.threads(); int from = thread_size * state.thread_index(); int to = from + thread_size; if (state.thread_index() == 0) { - test_vector = new std::vector(static_cast(size)); + test_vector = std::vector(static_cast(size)); } for (auto _ : state) { @@ -175,12 +192,12 @@ static void BM_ParallelMemset(benchmark::State& state) { } if (state.thread_index() == 0) { - delete test_vector; + test_vector.reset(); } } BENCHMARK(BM_ParallelMemset)->Arg(10 << 20)->ThreadRange(1, 4); -static void BM_ManualTiming(benchmark::State& state) { +void BM_ManualTiming(benchmark::State& state) { int64_t slept_for = 0; int64_t microseconds = state.range(0); std::chrono::duration sleep_duration{ @@ -204,8 +221,6 @@ static void BM_ManualTiming(benchmark::State& state) { BENCHMARK(BM_ManualTiming)->Range(1, 1 << 14)->UseRealTime(); BENCHMARK(BM_ManualTiming)->Range(1, 1 << 14)->UseManualTime(); -#ifdef BENCHMARK_HAS_CXX11 - template void BM_with_args(benchmark::State& state, Args&&...) { for (auto _ : state) { @@ -221,9 +236,32 @@ void BM_non_template_args(benchmark::State& state, int, double) { } BENCHMARK_CAPTURE(BM_non_template_args, basic_test, 0, 0); -#endif // BENCHMARK_HAS_CXX11 +template +void BM_template2_capture(benchmark::State& state, ExtraArgs&&... extra_args) { + static_assert(std::is_same::value, ""); + static_assert(std::is_same::value, ""); + static_assert(std::is_same::value, ""); + unsigned int dummy[sizeof...(ExtraArgs)] = {extra_args...}; + assert(dummy[0] == 42); + for (auto _ : state) { + } +} +BENCHMARK_TEMPLATE2_CAPTURE(BM_template2_capture, void, char*, foo, 42U); +BENCHMARK_CAPTURE((BM_template2_capture), foo, 42U); + +template +void BM_template1_capture(benchmark::State& state, ExtraArgs&&... extra_args) { + static_assert(std::is_same::value, ""); + static_assert(std::is_same::value, ""); + unsigned long dummy[sizeof...(ExtraArgs)] = {extra_args...}; + assert(dummy[0] == 24); + for (auto _ : state) { + } +} +BENCHMARK_TEMPLATE1_CAPTURE(BM_template1_capture, void, foo, 24UL); +BENCHMARK_CAPTURE(BM_template1_capture, foo, 24UL); -static void BM_DenseThreadRanges(benchmark::State& st) { +void BM_DenseThreadRanges(benchmark::State& st) { switch (st.range(0)) { case 1: assert(st.threads() == 1 || st.threads() == 2 || st.threads() == 3); @@ -245,4 +283,26 @@ BENCHMARK(BM_DenseThreadRanges)->Arg(1)->DenseThreadRange(1, 3); BENCHMARK(BM_DenseThreadRanges)->Arg(2)->DenseThreadRange(1, 4, 2); BENCHMARK(BM_DenseThreadRanges)->Arg(3)->DenseThreadRange(5, 14, 3); +void BM_BenchmarkName(benchmark::State& state) { + for (auto _ : state) { + } + + // Check that the benchmark name is passed correctly to `state`. + assert("BM_BenchmarkName" == state.name()); +} +BENCHMARK(BM_BenchmarkName); + +// regression test for #1446 +template +void BM_templated_test(benchmark::State& state) { + for (auto _ : state) { + type created_string; + benchmark::DoNotOptimize(created_string); + } +} + +const auto BM_templated_test_double = BM_templated_test>; +BENCHMARK(BM_templated_test_double); +} // end namespace + BENCHMARK_MAIN(); diff --git a/test/clobber_memory_assembly_test.cc b/test/clobber_memory_assembly_test.cc index ab269130cd..24e06660e4 100644 --- a/test/clobber_memory_assembly_test.cc +++ b/test/clobber_memory_assembly_test.cc @@ -1,8 +1,10 @@ -#include +#include "benchmark/macros.h" +#include "benchmark/utils.h" #ifdef __clang__ #pragma clang diagnostic ignored "-Wreturn-type" #endif +BENCHMARK_DISABLE_DEPRECATED_WARNING extern "C" { diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 3e14c4f941..64a7e72a6d 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -1,22 +1,26 @@ #undef NDEBUG -#include #include #include #include #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/statistics.h" +#include "benchmark/types.h" +#include "benchmark/utils.h" #include "output_test.h" namespace { #define ADD_COMPLEXITY_CASES(...) \ - int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__) + const int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__) -int AddComplexityTest(const std::string &test_name, - const std::string &big_o_test_name, - const std::string &rms_test_name, - const std::string &big_o, int family_index) { +int AddComplexityTest(const std::string& test_name, + const std::string& big_o_test_name, + const std::string& rms_test_name, + const std::string& big_o, int family_index) { SetSubstitutions({{"%name", test_name}, {"%bigo_name", big_o_test_name}, {"%rms_name", rms_test_name}, @@ -61,43 +65,50 @@ int AddComplexityTest(const std::string &test_name, return 0; } -} // end namespace - // ========================================================================= // // --------------------------- Testing BigO O(1) --------------------------- // // ========================================================================= // -void BM_Complexity_O1(benchmark::State &state) { +void BM_Complexity_O1(benchmark::State& state) { for (auto _ : state) { - for (int i = 0; i < 1024; ++i) { - benchmark::DoNotOptimize(&i); + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); + double tmp = static_cast(state.iterations()); + benchmark::DoNotOptimize(tmp); + for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) { + benchmark::DoNotOptimize(state.iterations()); + tmp *= static_cast(state.iterations()); + benchmark::DoNotOptimize(tmp); } + + // always 1ns per iteration + state.SetIterationTime(42 * 1e-9); } state.SetComplexityN(state.range(0)); } -BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1); -BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(); BENCHMARK(BM_Complexity_O1) ->Range(1, 1 << 18) + ->UseManualTime() + ->Complexity(benchmark::o1); +BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->UseManualTime()->Complexity(); +BENCHMARK(BM_Complexity_O1) + ->Range(1, 1 << 18) + ->UseManualTime() ->Complexity([](benchmark::IterationCount) { return 1.0; }); -const char *one_test_name = "BM_Complexity_O1"; -const char *big_o_1_test_name = "BM_Complexity_O1_BigO"; -const char *rms_o_1_test_name = "BM_Complexity_O1_RMS"; -const char *enum_big_o_1 = "\\([0-9]+\\)"; -// FIXME: Tolerate both '(1)' and 'lgN' as output when the complexity is auto -// deduced. -// See https://github.com/google/benchmark/issues/272 -const char *auto_big_o_1 = "(\\([0-9]+\\))|(lgN)"; -const char *lambda_big_o_1 = "f\\(N\\)"; +constexpr char one_test_name[] = "BM_Complexity_O1/manual_time"; +constexpr char big_o_1_test_name[] = "BM_Complexity_O1/manual_time_BigO"; +constexpr char rms_o_1_test_name[] = "BM_Complexity_O1/manual_time_RMS"; +constexpr char enum_auto_big_o_1[] = "\\([0-9]+\\)"; +constexpr char lambda_big_o_1[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_big_o_1, /*family_index=*/0); + enum_auto_big_o_1, /*family_index=*/0); -// Add auto enum tests +// Add auto tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - auto_big_o_1, /*family_index=*/1); + enum_auto_big_o_1, /*family_index=*/1); // Add lambda tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, @@ -107,120 +118,161 @@ ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, // --------------------------- Testing BigO O(N) --------------------------- // // ========================================================================= // -std::vector ConstructRandomVector(int64_t size) { - std::vector v; - v.reserve(static_cast(size)); - for (int i = 0; i < size; ++i) { - v.push_back(static_cast(std::rand() % size)); - } - return v; -} - -void BM_Complexity_O_N(benchmark::State &state) { - auto v = ConstructRandomVector(state.range(0)); - // Test worst case scenario (item not in vector) - const int64_t item_not_in_vector = state.range(0) * 2; +void BM_Complexity_O_N(benchmark::State& state) { for (auto _ : state) { - benchmark::DoNotOptimize(std::find(v.begin(), v.end(), item_not_in_vector)); + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); + double tmp = static_cast(state.iterations()); + benchmark::DoNotOptimize(tmp); + for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) { + benchmark::DoNotOptimize(state.iterations()); + tmp *= static_cast(state.iterations()); + benchmark::DoNotOptimize(tmp); + } + + // 1ns per iteration per entry + state.SetIterationTime(static_cast(state.range(0)) * 42 * 1e-9); } state.SetComplexityN(state.range(0)); } BENCHMARK(BM_Complexity_O_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) + ->Range(1 << 10, 1 << 20) + ->UseManualTime() ->Complexity(benchmark::oN); BENCHMARK(BM_Complexity_O_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) + ->Range(1 << 10, 1 << 20) + ->UseManualTime() + ->Complexity(); +BENCHMARK(BM_Complexity_O_N) + ->RangeMultiplier(2) + ->Range(1 << 10, 1 << 20) + ->UseManualTime() ->Complexity([](benchmark::IterationCount n) -> double { return static_cast(n); }); -BENCHMARK(BM_Complexity_O_N) - ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) - ->Complexity(); -const char *n_test_name = "BM_Complexity_O_N"; -const char *big_o_n_test_name = "BM_Complexity_O_N_BigO"; -const char *rms_o_n_test_name = "BM_Complexity_O_N_RMS"; -const char *enum_auto_big_o_n = "N"; -const char *lambda_big_o_n = "f\\(N\\)"; +constexpr char n_test_name[] = "BM_Complexity_O_N/manual_time"; +constexpr char big_o_n_test_name[] = "BM_Complexity_O_N/manual_time_BigO"; +constexpr char rms_o_n_test_name[] = "BM_Complexity_O_N/manual_time_RMS"; +constexpr char enum_auto_big_o_n[] = "N"; +constexpr char lambda_big_o_n[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n, /*family_index=*/3); +// Add auto tests +ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, + enum_auto_big_o_n, /*family_index=*/4); + // Add lambda tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - lambda_big_o_n, /*family_index=*/4); + lambda_big_o_n, /*family_index=*/5); // ========================================================================= // -// ------------------------- Testing BigO O(N*lgN) ------------------------- // +// ------------------------- Testing BigO O(NlgN) ------------------------- // // ========================================================================= // -static void BM_Complexity_O_N_log_N(benchmark::State &state) { - auto v = ConstructRandomVector(state.range(0)); +const double kLog2E = 1.44269504088896340736; +void BM_Complexity_O_N_log_N(benchmark::State& state) { for (auto _ : state) { - std::sort(v.begin(), v.end()); + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); + double tmp = static_cast(state.iterations()); + benchmark::DoNotOptimize(tmp); + for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) { + benchmark::DoNotOptimize(state.iterations()); + tmp *= static_cast(state.iterations()); + benchmark::DoNotOptimize(tmp); + } + + state.SetIterationTime(static_cast(state.range(0)) * kLog2E * + std::log(state.range(0)) * 42 * 1e-9); } state.SetComplexityN(state.range(0)); } -static const double kLog2E = 1.44269504088896340736; BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) + ->Range(1 << 10, 1U << 24) + ->UseManualTime() ->Complexity(benchmark::oNLogN); BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) - ->Complexity([](benchmark::IterationCount n) { - return kLog2E * static_cast(n) * log(static_cast(n)); - }); + ->Range(1 << 10, 1U << 24) + ->UseManualTime() + ->Complexity(); BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) - ->Complexity(); + ->Range(1 << 10, 1U << 24) + ->UseManualTime() + ->Complexity([](benchmark::IterationCount n) { + return kLog2E * static_cast(n) * std::log(static_cast(n)); + }); -const char *n_lg_n_test_name = "BM_Complexity_O_N_log_N"; -const char *big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO"; -const char *rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_RMS"; -const char *enum_auto_big_o_n_lg_n = "NlgN"; -const char *lambda_big_o_n_lg_n = "f\\(N\\)"; +constexpr char n_lg_n_test_name[] = "BM_Complexity_O_N_log_N/manual_time"; +constexpr char big_o_n_lg_n_test_name[] = + "BM_Complexity_O_N_log_N/manual_time_BigO"; +constexpr char rms_o_n_lg_n_test_name[] = + "BM_Complexity_O_N_log_N/manual_time_RMS"; +constexpr char enum_auto_big_o_n_lg_n[] = "NlgN"; +constexpr char lambda_big_o_n_lg_n[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, /*family_index=*/6); -// Add lambda tests +// NOTE: auto big-o is wron.g ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, - rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n, + rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, /*family_index=*/7); +//// Add lambda tests +ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, + rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n, + /*family_index=*/8); + // ========================================================================= // // -------- Testing formatting of Complexity with captured args ------------ // // ========================================================================= // -void BM_ComplexityCaptureArgs(benchmark::State &state, int n) { +void BM_ComplexityCaptureArgs(benchmark::State& state, int n) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero benchmark::DoNotOptimize(state.iterations()); + double tmp = static_cast(state.iterations()); + benchmark::DoNotOptimize(tmp); + for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) { + benchmark::DoNotOptimize(state.iterations()); + tmp *= static_cast(state.iterations()); + benchmark::DoNotOptimize(tmp); + } + + state.SetIterationTime(static_cast(state.range(0)) * 42 * 1e-9); } state.SetComplexityN(n); } BENCHMARK_CAPTURE(BM_ComplexityCaptureArgs, capture_test, 100) + ->UseManualTime() ->Complexity(benchmark::oN) ->Ranges({{1, 2}, {3, 4}}); const std::string complexity_capture_name = - "BM_ComplexityCaptureArgs/capture_test"; + "BM_ComplexityCaptureArgs/capture_test/manual_time"; ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO", - complexity_capture_name + "_RMS", "N", /*family_index=*/9); + complexity_capture_name + "_RMS", "N", + /*family_index=*/9); +} // end namespace // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // // ========================================================================= // -int main(int argc, char *argv[]) { RunOutputTests(argc, argv); } +int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + RunOutputTests(argc, argv); +} diff --git a/test/cxx03_test.cc b/test/cxx03_test.cc deleted file mode 100644 index 9711c1bd4a..0000000000 --- a/test/cxx03_test.cc +++ /dev/null @@ -1,62 +0,0 @@ -#undef NDEBUG -#include -#include - -#include "benchmark/benchmark.h" - -#if __cplusplus >= 201103L -#error C++11 or greater detected. Should be C++03. -#endif - -#ifdef BENCHMARK_HAS_CXX11 -#error C++11 or greater detected by the library. BENCHMARK_HAS_CXX11 is defined. -#endif - -void BM_empty(benchmark::State& state) { - while (state.KeepRunning()) { - volatile benchmark::IterationCount x = state.iterations(); - ((void)x); - } -} -BENCHMARK(BM_empty); - -// The new C++11 interface for args/ranges requires initializer list support. -// Therefore we provide the old interface to support C++03. -void BM_old_arg_range_interface(benchmark::State& state) { - assert((state.range(0) == 1 && state.range(1) == 2) || - (state.range(0) == 5 && state.range(1) == 6)); - while (state.KeepRunning()) { - } -} -BENCHMARK(BM_old_arg_range_interface)->ArgPair(1, 2)->RangePair(5, 5, 6, 6); - -template -void BM_template2(benchmark::State& state) { - BM_empty(state); -} -BENCHMARK_TEMPLATE2(BM_template2, int, long); - -template -void BM_template1(benchmark::State& state) { - BM_empty(state); -} -BENCHMARK_TEMPLATE(BM_template1, long); -BENCHMARK_TEMPLATE1(BM_template1, int); - -template -struct BM_Fixture : public ::benchmark::Fixture {}; - -BENCHMARK_TEMPLATE_F(BM_Fixture, BM_template1, long)(benchmark::State& state) { - BM_empty(state); -} -BENCHMARK_TEMPLATE1_F(BM_Fixture, BM_template2, int)(benchmark::State& state) { - BM_empty(state); -} - -void BM_counters(benchmark::State& state) { - BM_empty(state); - state.counters["Foo"] = 2; -} -BENCHMARK(BM_counters); - -BENCHMARK_MAIN(); diff --git a/test/cxx11_test.cc b/test/cxx11_test.cc new file mode 100644 index 0000000000..a2e8bc895f --- /dev/null +++ b/test/cxx11_test.cc @@ -0,0 +1,12 @@ +#include "benchmark/benchmark_api.h" + +#if defined(_MSC_VER) +#if _MSVC_LANG != 201402L +// MSVC, even in C++11 mode, dooes not claim to be in C++11 mode. +#error "Trying to compile C++11 test with wrong C++ standard" +#endif // _MSVC_LANG +#else // Non-MSVC +#if __cplusplus != 201103L +#error "Trying to compile C++11 test with wrong C++ standard" +#endif // Non-MSVC +#endif diff --git a/test/diagnostics_test.cc b/test/diagnostics_test.cc index c54d5b0d70..a79e49f49c 100644 --- a/test/diagnostics_test.cc +++ b/test/diagnostics_test.cc @@ -11,12 +11,16 @@ #include #include "../src/check.h" -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/utils.h" #if defined(__GNUC__) && !defined(__EXCEPTIONS) #define TEST_HAS_NO_EXCEPTIONS #endif +namespace { void TestHandler() { #ifndef TEST_HAS_NO_EXCEPTIONS throw std::logic_error(""); @@ -46,13 +50,19 @@ void try_invalid_pause_resume(benchmark::State& state) { void BM_diagnostic_test(benchmark::State& state) { static bool called_once = false; - if (called_once == false) try_invalid_pause_resume(state); + if (!called_once) { + try_invalid_pause_resume(state); + } for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } - if (called_once == false) try_invalid_pause_resume(state); + if (!called_once) { + try_invalid_pause_resume(state); + } called_once = true; } @@ -61,20 +71,37 @@ BENCHMARK(BM_diagnostic_test); void BM_diagnostic_test_keep_running(benchmark::State& state) { static bool called_once = false; - if (called_once == false) try_invalid_pause_resume(state); + if (!called_once) { + try_invalid_pause_resume(state); + } while (state.KeepRunning()) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } - if (called_once == false) try_invalid_pause_resume(state); + if (!called_once) { + try_invalid_pause_resume(state); + } called_once = true; } BENCHMARK(BM_diagnostic_test_keep_running); +} // end namespace int main(int argc, char* argv[]) { +#ifdef NDEBUG + // This test is exercising functionality for debug builds, which are not + // available in release builds. Skip the test if we are in that environment + // to avoid a test failure. + std::cout << "Diagnostic test disabled in release build\n"; + (void)argc; + (void)argv; +#else + benchmark::MaybeReenterWithoutASLR(argc, argv); benchmark::internal::GetAbortHandler() = &TestHandler; benchmark::Initialize(&argc, argv); benchmark::RunSpecifiedBenchmarks(); +#endif } diff --git a/test/display_aggregates_only_test.cc b/test/display_aggregates_only_test.cc index 6ad65e7f51..86f202fe5b 100644 --- a/test/display_aggregates_only_test.cc +++ b/test/display_aggregates_only_test.cc @@ -3,20 +3,26 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" #include "output_test.h" // Ok this test is super ugly. We want to check what happens with the file // reporter in the presence of DisplayAggregatesOnly(). // We do not care about console output, the normal tests check that already. +namespace { void BM_SummaryRepeat(benchmark::State& state) { for (auto _ : state) { } } BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->DisplayAggregatesOnly(); +} // end namespace int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + const std::string output = GetFileReporterOutput(argc, argv); if (SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3") != 7 || diff --git a/test/donotoptimize_assembly_test.cc b/test/donotoptimize_assembly_test.cc index 70e780a5f0..d7b3b549d2 100644 --- a/test/donotoptimize_assembly_test.cc +++ b/test/donotoptimize_assembly_test.cc @@ -1,8 +1,11 @@ -#include +#include "benchmark/macros.h" +#include "benchmark/utils.h" #ifdef __clang__ #pragma clang diagnostic ignored "-Wreturn-type" +#pragma clang diagnostic ignored "-Wmissing-prototypes" #endif +BENCHMARK_DISABLE_DEPRECATED_WARNING extern "C" { @@ -18,7 +21,7 @@ inline int Add42(int x) { return x + 42; } struct NotTriviallyCopyable { NotTriviallyCopyable(); explicit NotTriviallyCopyable(int x) : value(x) {} - NotTriviallyCopyable(NotTriviallyCopyable const &); + NotTriviallyCopyable(NotTriviallyCopyable const&); int value; }; @@ -184,7 +187,7 @@ extern "C" void test_pointer_const_lvalue() { // CHECK-CLANG: movq %rax, -{{[0-9]+}}(%[[REG:[a-z]+]]) // CHECK: ret int x = 42; - int *const xp = &x; + int* const xp = &x; benchmark::DoNotOptimize(xp); } @@ -195,6 +198,6 @@ extern "C" void test_pointer_lvalue() { // CHECK-CLANG: movq %rax, -{{[0-9]+}}(%[[REG:[a-z+]+]]) // CHECK: ret int x = 42; - int *xp = &x; + int* xp = &x; benchmark::DoNotOptimize(xp); } diff --git a/test/donotoptimize_test.cc b/test/donotoptimize_test.cc index 5c0d3b6eac..34b1ed6479 100644 --- a/test/donotoptimize_test.cc +++ b/test/donotoptimize_test.cc @@ -1,10 +1,11 @@ #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/utils.h" namespace { #if defined(__GNUC__) -std::int64_t double_up(const std::int64_t x) __attribute__((const)); +std::int64_t double_up(std::int64_t x) __attribute__((const)); #endif std::int64_t double_up(const std::int64_t x) { return x * 2; } } // namespace @@ -26,9 +27,20 @@ struct BitRef { BitRef(int i, unsigned char& b) : index(i), byte(b) {} }; -int main(int, char*[]) { +int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + // this test verifies compilation of DoNotOptimize() for some types + char buffer1[1] = ""; + benchmark::DoNotOptimize(buffer1); + + char buffer2[2] = ""; + benchmark::DoNotOptimize(buffer2); + + char buffer3[3] = ""; + benchmark::DoNotOptimize(buffer3); + char buffer8[8] = ""; benchmark::DoNotOptimize(buffer8); @@ -37,17 +49,22 @@ int main(int, char*[]) { char buffer1024[1024] = ""; benchmark::DoNotOptimize(buffer1024); - benchmark::DoNotOptimize(&buffer1024[0]); + char* bptr = &buffer1024[0]; + benchmark::DoNotOptimize(bptr); int x = 123; benchmark::DoNotOptimize(x); - benchmark::DoNotOptimize(&x); + int* xp = &x; + benchmark::DoNotOptimize(xp); benchmark::DoNotOptimize(x += 42); - benchmark::DoNotOptimize(double_up(x)); + std::int64_t y = double_up(x); + benchmark::DoNotOptimize(y); // These tests are to e - benchmark::DoNotOptimize(BitRef::Make()); BitRef lval = BitRef::Make(); benchmark::DoNotOptimize(lval); + + // Check that accept rvalue. + benchmark::DoNotOptimize(BitRef::Make()); } diff --git a/test/filter_test.cc b/test/filter_test.cc index 266584a0f1..bcbf2ff6de 100644 --- a/test/filter_test.cc +++ b/test/filter_test.cc @@ -8,17 +8,20 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/reporter.h" +#include "benchmark/state.h" namespace { class TestReporter : public benchmark::ConsoleReporter { public: - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { + bool ReportContext(const Context& context) override { return ConsoleReporter::ReportContext(context); }; - virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + void ReportRuns(const std::vector& report) override { ++count_; max_family_index_ = std::max(max_family_index_, report[0].family_index); ConsoleReporter::ReportRuns(report); @@ -26,7 +29,7 @@ class TestReporter : public benchmark::ConsoleReporter { TestReporter() : count_(0), max_family_index_(0) {} - virtual ~TestReporter() {} + ~TestReporter() override {} int GetCount() const { return count_; } @@ -37,43 +40,45 @@ class TestReporter : public benchmark::ConsoleReporter { mutable int64_t max_family_index_; }; -} // end namespace - -static void NoPrefix(benchmark::State& state) { +void NoPrefix(benchmark::State& state) { for (auto _ : state) { } } BENCHMARK(NoPrefix); -static void BM_Foo(benchmark::State& state) { +void BM_Foo(benchmark::State& state) { for (auto _ : state) { } } BENCHMARK(BM_Foo); -static void BM_Bar(benchmark::State& state) { +void BM_Bar(benchmark::State& state) { for (auto _ : state) { } } BENCHMARK(BM_Bar); -static void BM_FooBar(benchmark::State& state) { +void BM_FooBar(benchmark::State& state) { for (auto _ : state) { } } BENCHMARK(BM_FooBar); -static void BM_FooBa(benchmark::State& state) { +void BM_FooBa(benchmark::State& state) { for (auto _ : state) { } } BENCHMARK(BM_FooBa); +} // end namespace int main(int argc, char** argv) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + bool list_only = false; - for (int i = 0; i < argc; ++i) + for (int i = 0; i < argc; ++i) { list_only |= std::string(argv[i]).find("--benchmark_list_tests") != std::string::npos; + } benchmark::Initialize(&argc, argv); @@ -84,13 +89,13 @@ int main(int argc, char** argv) { if (argc == 2) { // Make sure we ran all of the tests std::stringstream ss(argv[1]); - int64_t expected_return; + int64_t expected_return = 0; ss >> expected_return; if (returned_count != expected_return) { std::cerr << "ERROR: Expected " << expected_return << " tests to match the filter but returned_count = " - << returned_count << std::endl; + << returned_count << '\n'; return -1; } @@ -99,7 +104,7 @@ int main(int argc, char** argv) { if (reports_count != expected_reports) { std::cerr << "ERROR: Expected " << expected_reports << " tests to be run but reported_count = " << reports_count - << std::endl; + << '\n'; return -1; } @@ -108,7 +113,7 @@ int main(int argc, char** argv) { if (num_families != expected_reports) { std::cerr << "ERROR: Expected " << expected_reports << " test families to be run but num_families = " - << num_families << std::endl; + << num_families << '\n'; return -1; } } diff --git a/test/fixture_test.cc b/test/fixture_test.cc index af650dbd06..8994dee1c1 100644 --- a/test/fixture_test.cc +++ b/test/fixture_test.cc @@ -2,27 +2,29 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" #define FIXTURE_BECHMARK_NAME MyFixture class FIXTURE_BECHMARK_NAME : public ::benchmark::Fixture { public: - void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE { + void SetUp(const ::benchmark::State& state) override { if (state.thread_index() == 0) { assert(data.get() == nullptr); data.reset(new int(42)); } } - void TearDown(const ::benchmark::State& state) BENCHMARK_OVERRIDE { + void TearDown(const ::benchmark::State& state) override { if (state.thread_index() == 0) { assert(data.get() != nullptr); data.reset(); } } - ~FIXTURE_BECHMARK_NAME() { assert(data == nullptr); } + ~FIXTURE_BECHMARK_NAME() override { assert(data == nullptr); } std::unique_ptr data; }; diff --git a/test/internal_threading_test.cc b/test/internal_threading_test.cc index 62b5b955a9..4d1dd1b711 100644 --- a/test/internal_threading_test.cc +++ b/test/internal_threading_test.cc @@ -5,11 +5,15 @@ #include #include "../src/timers.h" -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/counter.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" #include "output_test.h" -static const std::chrono::duration time_frame(50); -static const double time_frame_in_sec( +namespace { +const std::chrono::duration time_frame(50); +const double time_frame_in_sec( std::chrono::duration_cast>>( time_frame) .count()); @@ -22,8 +26,9 @@ void MyBusySpinwait() { const auto elapsed = now - start; if (std::chrono::duration(elapsed) >= - time_frame) + time_frame) { return; + } } } @@ -177,9 +182,14 @@ BENCHMARK(BM_MainThreadAndWorkerThread) ->Threads(2) ->MeasureProcessCPUTime() ->UseManualTime(); +} // end namespace // ========================================================================= // // ---------------------------- TEST CASES END ----------------------------- // // ========================================================================= // -int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } +int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + + RunOutputTests(argc, argv); +} diff --git a/test/link_main_test.cc b/test/link_main_test.cc index 241ad5c390..538f807020 100644 --- a/test/link_main_test.cc +++ b/test/link_main_test.cc @@ -1,8 +1,14 @@ -#include "benchmark/benchmark.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/utils.h" +namespace { void BM_empty(benchmark::State& state) { for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } } BENCHMARK(BM_empty); +} // end namespace diff --git a/test/locale_impermeability_test.cc b/test/locale_impermeability_test.cc new file mode 100644 index 0000000000..98982a6505 --- /dev/null +++ b/test/locale_impermeability_test.cc @@ -0,0 +1,50 @@ +#undef NDEBUG +#include +#include +#include + +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "output_test.h" + +namespace { +void BM_ostream(benchmark::State& state) { +#if !defined(__GLIBCXX__) || defined(__GLIBC__) || defined(__DragonFly__) + // libstdc++ only supports locale manipulations on GNU and + // DragonflyBSD platforms at the time of writing, don't run the test + // on other platforms when using libstdc++. + std::locale::global(std::locale("en_US.UTF-8")); +#endif + while (state.KeepRunning()) { + state.SetIterationTime(1e-6); + } +} +BENCHMARK(BM_ostream)->UseManualTime()->Iterations(1000000); + +ADD_CASES(TC_ConsoleOut, {{"^BM_ostream/iterations:1000000/manual_time" + " %console_report$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_ostream/iterations:1000000/manual_time\",$"}, + {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": " + "\"BM_ostream/iterations:1000000/manual_time\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": 1000000,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_ostream/iterations:1000000/" + "manual_time\",1000000,%float,%float,ns,,,,,$"}}); +} // end namespace + +int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + RunOutputTests(argc, argv); +} diff --git a/test/manual_threading_test.cc b/test/manual_threading_test.cc new file mode 100644 index 0000000000..bac36bce95 --- /dev/null +++ b/test/manual_threading_test.cc @@ -0,0 +1,178 @@ + +#include +#undef NDEBUG + +#include +#include + +#include "../src/timers.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/counter.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" + +namespace { + +const std::chrono::duration time_frame(50); +const double time_frame_in_sec( + std::chrono::duration_cast>>( + time_frame) + .count()); + +void MyBusySpinwait() { + const auto start = benchmark::ChronoClockNow(); + + while (true) { + const auto now = benchmark::ChronoClockNow(); + const auto elapsed = now - start; + + if (std::chrono::duration(elapsed) >= + time_frame) { + return; + } + } +} + +int numRunThreadsCalled_ = 0; + +class ManualThreadRunner : public benchmark::ThreadRunnerBase { + public: + explicit ManualThreadRunner(int num_threads) + : pool(static_cast(num_threads - 1)) {} + + void RunThreads(const std::function& fn) final { + for (std::size_t ti = 0; ti < pool.size(); ++ti) { + pool[ti] = std::thread(fn, static_cast(ti + 1)); + } + + fn(0); + + for (std::thread& thread : pool) { + thread.join(); + } + + ++numRunThreadsCalled_; + } + + private: + std::vector pool; +}; + +// ========================================================================= // +// --------------------------- TEST CASES BEGIN ---------------------------- // +// ========================================================================= // + +// ========================================================================= // +// BM_ManualThreading +// Creation of threads is done before the start of the measurement, +// joining after the finish of the measurement. +void BM_ManualThreading(benchmark::State& state) { + for (auto _ : state) { + MyBusySpinwait(); + state.SetIterationTime(time_frame_in_sec); + } + state.counters["invtime"] = + benchmark::Counter{1, benchmark::Counter::kIsRate}; +} + +} // end namespace + +BENCHMARK(BM_ManualThreading) + ->Iterations(1) + ->ThreadRunner([](int num_threads) { + return std::make_unique(num_threads); + }) + ->Threads(1); +BENCHMARK(BM_ManualThreading) + ->Iterations(1) + ->ThreadRunner([](int num_threads) { + return std::make_unique(num_threads); + }) + ->Threads(1) + ->UseRealTime(); +BENCHMARK(BM_ManualThreading) + ->Iterations(1) + ->ThreadRunner([](int num_threads) { + return std::make_unique(num_threads); + }) + ->Threads(1) + ->UseManualTime(); +BENCHMARK(BM_ManualThreading) + ->Iterations(1) + ->ThreadRunner([](int num_threads) { + return std::make_unique(num_threads); + }) + ->Threads(1) + ->MeasureProcessCPUTime(); +BENCHMARK(BM_ManualThreading) + ->Iterations(1) + ->ThreadRunner([](int num_threads) { + return std::make_unique(num_threads); + }) + ->Threads(1) + ->MeasureProcessCPUTime() + ->UseRealTime(); +BENCHMARK(BM_ManualThreading) + ->Iterations(1) + ->ThreadRunner([](int num_threads) { + return std::make_unique(num_threads); + }) + ->Threads(1) + ->MeasureProcessCPUTime() + ->UseManualTime(); + +BENCHMARK(BM_ManualThreading) + ->Iterations(1) + ->ThreadRunner([](int num_threads) { + return std::make_unique(num_threads); + }) + ->Threads(2); +BENCHMARK(BM_ManualThreading) + ->Iterations(1) + ->ThreadRunner([](int num_threads) { + return std::make_unique(num_threads); + }) + ->Threads(2) + ->UseRealTime(); +BENCHMARK(BM_ManualThreading) + ->Iterations(1) + ->ThreadRunner([](int num_threads) { + return std::make_unique(num_threads); + }) + ->Threads(2) + ->UseManualTime(); +BENCHMARK(BM_ManualThreading) + ->Iterations(1) + ->ThreadRunner([](int num_threads) { + return std::make_unique(num_threads); + }) + ->Threads(2) + ->MeasureProcessCPUTime(); +BENCHMARK(BM_ManualThreading) + ->Iterations(1) + ->ThreadRunner([](int num_threads) { + return std::make_unique(num_threads); + }) + ->Threads(2) + ->MeasureProcessCPUTime() + ->UseRealTime(); +BENCHMARK(BM_ManualThreading) + ->Iterations(1) + ->ThreadRunner([](int num_threads) { + return std::make_unique(num_threads); + }) + ->Threads(2) + ->MeasureProcessCPUTime() + ->UseManualTime(); + +// ========================================================================= // +// ---------------------------- TEST CASES END ----------------------------- // +// ========================================================================= // + +int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + benchmark::Initialize(&argc, argv); + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); + assert(numRunThreadsCalled_ > 0); +} diff --git a/test/map_test.cc b/test/map_test.cc index 509613457c..f4b41d2a07 100644 --- a/test/map_test.cc +++ b/test/map_test.cc @@ -1,7 +1,10 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/utils.h" namespace { @@ -13,10 +16,8 @@ std::map ConstructRandomMap(int size) { return m; } -} // namespace - // Basic version. -static void BM_MapLookup(benchmark::State& state) { +void BM_MapLookup(benchmark::State& state) { const int size = static_cast(state.range(0)); std::map m; for (auto _ : state) { @@ -24,21 +25,23 @@ static void BM_MapLookup(benchmark::State& state) { m = ConstructRandomMap(size); state.ResumeTiming(); for (int i = 0; i < size; ++i) { - benchmark::DoNotOptimize(m.find(std::rand() % size)); + auto it = m.find(std::rand() % size); + benchmark::DoNotOptimize(it); } } state.SetItemsProcessed(state.iterations() * size); } BENCHMARK(BM_MapLookup)->Range(1 << 3, 1 << 12); +} // namespace // Using fixtures. class MapFixture : public ::benchmark::Fixture { public: - void SetUp(const ::benchmark::State& st) BENCHMARK_OVERRIDE { + void SetUp(const ::benchmark::State& st) override { m = ConstructRandomMap(static_cast(st.range(0))); } - void TearDown(const ::benchmark::State&) BENCHMARK_OVERRIDE { m.clear(); } + void TearDown(const ::benchmark::State& /*unused*/) override { m.clear(); } std::map m; }; @@ -47,7 +50,8 @@ BENCHMARK_DEFINE_F(MapFixture, Lookup)(benchmark::State& state) { const int size = static_cast(state.range(0)); for (auto _ : state) { for (int i = 0; i < size; ++i) { - benchmark::DoNotOptimize(m.find(std::rand() % size)); + auto it = m.find(std::rand() % size); + benchmark::DoNotOptimize(it); } } state.SetItemsProcessed(state.iterations() * size); diff --git a/test/memory_manager_test.cc b/test/memory_manager_test.cc index f0c192fcbd..36de9d4ca2 100644 --- a/test/memory_manager_test.cc +++ b/test/memory_manager_test.cc @@ -1,23 +1,30 @@ #include -#include "../src/check.h" -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/managers.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/utils.h" #include "output_test.h" +namespace { class TestMemoryManager : public benchmark::MemoryManager { - void Start() BENCHMARK_OVERRIDE {} - void Stop(Result* result) BENCHMARK_OVERRIDE { - result->num_allocs = 42; - result->max_bytes_used = 42000; + void Start() override {} + void Stop(Result& result) override { + result.num_allocs = 42; + result.max_bytes_used = 42000; } }; void BM_empty(benchmark::State& state) { for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } } BENCHMARK(BM_empty); +} // end namespace ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, @@ -38,6 +45,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_empty\",%csv_report$"}}); int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); std::unique_ptr mm(new TestMemoryManager()); benchmark::RegisterMemoryManager(mm.get()); diff --git a/test/memory_results_gtest.cc b/test/memory_results_gtest.cc new file mode 100644 index 0000000000..f856fbf67b --- /dev/null +++ b/test/memory_results_gtest.cc @@ -0,0 +1,104 @@ +#include + +#include "benchmark/benchmark_api.h" +#include "benchmark/managers.h" +#include "benchmark/reporter.h" +#include "benchmark/state.h" +#include "gtest/gtest.h" + +namespace { + +using benchmark::Benchmark; +using benchmark::ClearRegisteredBenchmarks; +using benchmark::ConsoleReporter; +using benchmark::MemoryManager; +using benchmark::RegisterBenchmark; +using benchmark::RunSpecifiedBenchmarks; +using benchmark::State; + +constexpr int N_REPETITIONS = 100; +constexpr int N_ITERATIONS = 1; + +int num_allocs = 0; +int max_bytes_used = 0; +int total_allocated_bytes = 0; +int net_heap_growth = 0; + +void reset() { + num_allocs = 0; + max_bytes_used = 0; + total_allocated_bytes = 0; + net_heap_growth = 0; +} +class TestMemoryManager : public MemoryManager { + void Start() override {} + void Stop(Result& result) override { + result.num_allocs = num_allocs; + result.net_heap_growth = net_heap_growth; + result.max_bytes_used = max_bytes_used; + result.total_allocated_bytes = total_allocated_bytes; + + num_allocs += 1; + max_bytes_used += 2; + net_heap_growth += 4; + total_allocated_bytes += 10; + } +}; + +class TestReporter : public ConsoleReporter { + public: + TestReporter() = default; + virtual ~TestReporter() = default; + + bool ReportContext(const Context& /*unused*/) override { return true; } + + void PrintHeader(const Run&) override {} + void PrintRunData(const Run& run) override { + if (run.repetition_index == -1) return; + if (!run.memory_result.memory_iterations) return; + + store.push_back(run.memory_result); + } + + std::vector store; +}; + +class MemoryResultsTest : public testing::Test { + public: + Benchmark* bm; + TestReporter reporter; + + void SetUp() override { + bm = RegisterBenchmark("BM", [](State& st) { + for (auto _ : st) { + } + }); + bm->Repetitions(N_REPETITIONS); + bm->Iterations(N_ITERATIONS); + reset(); + } + void TearDown() override { ClearRegisteredBenchmarks(); } +}; + +TEST_F(MemoryResultsTest, NoMMTest) { + RunSpecifiedBenchmarks(&reporter); + EXPECT_EQ(reporter.store.size(), 0); +} + +TEST_F(MemoryResultsTest, ResultsTest) { + auto mm = std::make_unique(); + RegisterMemoryManager(mm.get()); + + RunSpecifiedBenchmarks(&reporter); + EXPECT_EQ(reporter.store.size(), N_REPETITIONS); + + for (size_t i = 0; i < reporter.store.size(); i++) { + EXPECT_EQ(reporter.store[i].num_allocs, static_cast(i)); + EXPECT_EQ(reporter.store[i].max_bytes_used, static_cast(i) * 2); + EXPECT_EQ(reporter.store[i].net_heap_growth, static_cast(i) * 4); + EXPECT_EQ(reporter.store[i].total_allocated_bytes, + static_cast(i) * 10); + } +} + +} // namespace diff --git a/test/min_time_parse_gtest.cc b/test/min_time_parse_gtest.cc new file mode 100644 index 0000000000..e2bdf67850 --- /dev/null +++ b/test/min_time_parse_gtest.cc @@ -0,0 +1,30 @@ +#include "../src/benchmark_runner.h" +#include "gtest/gtest.h" + +namespace { + +TEST(ParseMinTimeTest, InvalidInput) { +#if GTEST_HAS_DEATH_TEST + // Tests only runnable in debug mode (when BM_CHECK is enabled). +#ifndef NDEBUG +#ifndef TEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS + ASSERT_DEATH_IF_SUPPORTED( + { benchmark::internal::ParseBenchMinTime("abc"); }, + "Malformed seconds value passed to --benchmark_min_time: `abc`"); + + ASSERT_DEATH_IF_SUPPORTED( + { benchmark::internal::ParseBenchMinTime("123ms"); }, + "Malformed seconds value passed to --benchmark_min_time: `123ms`"); + + ASSERT_DEATH_IF_SUPPORTED( + { benchmark::internal::ParseBenchMinTime("1z"); }, + "Malformed seconds value passed to --benchmark_min_time: `1z`"); + + ASSERT_DEATH_IF_SUPPORTED( + { benchmark::internal::ParseBenchMinTime("1hs"); }, + "Malformed seconds value passed to --benchmark_min_time: `1hs`"); +#endif +#endif +#endif +} +} // namespace diff --git a/test/multiple_ranges_test.cc b/test/multiple_ranges_test.cc index 7618c4da08..695b52a9e1 100644 --- a/test/multiple_ranges_test.cc +++ b/test/multiple_ranges_test.cc @@ -3,8 +3,12 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/utils.h" +namespace { class MultipleRangesFixture : public ::benchmark::Fixture { public: MultipleRangesFixture() @@ -28,7 +32,7 @@ class MultipleRangesFixture : public ::benchmark::Fixture { {2, 7, 15}, {7, 6, 3}}) {} - void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE { + void SetUp(const ::benchmark::State& state) override { std::vector ranges = {state.range(0), state.range(1), state.range(2)}; @@ -39,7 +43,7 @@ class MultipleRangesFixture : public ::benchmark::Fixture { // NOTE: This is not TearDown as we want to check after _all_ runs are // complete. - virtual ~MultipleRangesFixture() { + ~MultipleRangesFixture() override { if (actualValues != expectedValues) { std::cout << "EXPECTED\n"; for (const auto& v : expectedValues) { @@ -87,10 +91,11 @@ void BM_CheckDefaultArgument(benchmark::State& state) { } BENCHMARK(BM_CheckDefaultArgument)->Ranges({{1, 5}, {6, 10}}); -static void BM_MultipleRanges(benchmark::State& st) { +void BM_MultipleRanges(benchmark::State& st) { for (auto _ : st) { } } BENCHMARK(BM_MultipleRanges)->Ranges({{5, 5}, {6, 6}}); +} // end namespace BENCHMARK_MAIN(); diff --git a/test/options_test.cc b/test/options_test.cc index 6bba0ea846..7ace93364b 100644 --- a/test/options_test.cc +++ b/test/options_test.cc @@ -1,13 +1,17 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/types.h" #if defined(NDEBUG) #undef NDEBUG #endif #include +namespace { void BM_basic(benchmark::State& state) { for (auto _ : state) { } @@ -50,7 +54,7 @@ BENCHMARK(BM_basic)->RangeMultiplier(4)->Range(-8, 8); BENCHMARK(BM_basic)->DenseRange(-2, 2, 1); BENCHMARK(BM_basic)->Ranges({{-64, 1}, {-8, -1}}); -void CustomArgs(benchmark::internal::Benchmark* b) { +void CustomArgs(benchmark::Benchmark* b) { for (int i = 0; i < 10; ++i) { b->Arg(i); } @@ -67,11 +71,12 @@ void BM_explicit_iteration_count(benchmark::State& state) { // Test that the requested iteration count is respected. assert(state.max_iterations == 42); - size_t actual_iterations = 0; - for (auto _ : state) ++actual_iterations; + for (auto _ : state) { + } assert(state.iterations() == state.max_iterations); assert(state.iterations() == 42); } BENCHMARK(BM_explicit_iteration_count)->Iterations(42); +} // end namespace BENCHMARK_MAIN(); diff --git a/test/output_test.h b/test/output_test.h index c6ff8ef2d3..9337edc07a 100644 --- a/test/output_test.h +++ b/test/output_test.h @@ -11,17 +11,18 @@ #include #include "../src/re.h" -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" #define CONCAT2(x, y) x##y #define CONCAT(x, y) CONCAT2(x, y) -#define ADD_CASES(...) int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__) +#define ADD_CASES(...) \ + const int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__) #define SET_SUBSTITUTIONS(...) \ - int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__) + const int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__) -enum MatchRules { +enum MatchRules : uint8_t { MR_Default, // Skip non-matching lines until a match is found. MR_Next, // Match must occur on the next line. MR_Not // No line between the current position and the next match matches @@ -37,7 +38,7 @@ struct TestCase { std::shared_ptr regex; }; -enum TestCaseID { +enum TestCaseID : uint8_t { TC_ConsoleOut, TC_ConsoleErr, TC_JSONOut, @@ -80,12 +81,13 @@ std::string GetFileReporterOutput(int argc, char* argv[]); // will be the subject of a call to checker_function // checker_function: should be of type ResultsCheckFn (see below) #define CHECK_BENCHMARK_RESULTS(bm_name_pattern, checker_function) \ - size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name_pattern, checker_function) + const size_t CONCAT(dummy, __LINE__) = \ + AddChecker(bm_name_pattern, checker_function) struct Results; typedef std::function ResultsCheckFn; -size_t AddChecker(const char* bm_name_pattern, const ResultsCheckFn& fn); +size_t AddChecker(const std::string& bm_name_pattern, const ResultsCheckFn& fn); // Class holding the results of a benchmark. // It is passed in calls to checker functions. @@ -101,7 +103,7 @@ struct Results { double NumIterations() const; - typedef enum { kCpuTime, kRealTime } BenchmarkTime; + typedef enum : uint8_t { kCpuTime, kRealTime } BenchmarkTime; // get cpu_time or real_time in seconds double GetTime(BenchmarkTime which) const; @@ -117,7 +119,7 @@ struct Results { // get the string for a result by name, or nullptr if the name // is not found - const std::string* Get(const char* entry_name) const { + const std::string* Get(const std::string& entry_name) const { auto it = values.find(entry_name); if (it == values.end()) return nullptr; return &it->second; @@ -126,12 +128,12 @@ struct Results { // get a result by name, parsed as a specific type. // NOTE: for counters, use GetCounterAs instead. template - T GetAs(const char* entry_name) const; + T GetAs(const std::string& entry_name) const; // counters are written as doubles, so they have to be read first // as a double, and only then converted to the asked type. template - T GetCounterAs(const char* entry_name) const { + T GetCounterAs(const std::string& entry_name) const { double dval = GetAs(entry_name); T tval = static_cast(dval); return tval; @@ -139,7 +141,7 @@ struct Results { }; template -T Results::GetAs(const char* entry_name) const { +T Results::GetAs(const std::string& entry_name) const { auto* sv = Get(entry_name); BM_CHECK(sv != nullptr && !sv->empty()); std::stringstream ss; diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index 81584cbf77..5dc42fb482 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -30,7 +30,7 @@ using TestCaseList = std::vector; using SubMap = std::vector>; TestCaseList& GetTestCaseList(TestCaseID ID) { - // Uses function-local statics to ensure initialization occurs + // Uses function-local static to ensure initialization occurs // before first use. static TestCaseList lists[TC_NumID]; return lists[ID]; @@ -45,7 +45,7 @@ SubMap& GetSubstitutions() { static SubMap map = { {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, // human-readable float - {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, + {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kKMGTPEZYmunpfazy]?i?"}, {"%percentage", percentage_re}, {"%int", "[ ]*[0-9]+"}, {" %s ", "[ ]+"}, @@ -65,6 +65,7 @@ SubMap& GetSubstitutions() { {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"}, {"%csv_ms_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ms,,,,,"}, {"%csv_s_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",s,,,,,"}, + {"%csv_cv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",,,,,,"}, {"%csv_bytes_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"}, {"%csv_items_report", @@ -82,7 +83,7 @@ std::string PerformSubstitutions(std::string source) { SubMap const& subs = GetSubstitutions(); using SizeT = std::string::size_type; for (auto const& KV : subs) { - SizeT pos; + SizeT pos = 0; SizeT next_start = 0; while ((pos = source.find(KV.first, next_start)) != std::string::npos) { next_start = pos + KV.second.size(); @@ -97,7 +98,7 @@ void CheckCase(std::stringstream& remaining_output, TestCase const& TC, std::string first_line; bool on_first = true; std::string line; - while (remaining_output.eof() == false) { + while (!remaining_output.eof()) { BM_CHECK(remaining_output.good()); std::getline(remaining_output, line); if (on_first) { @@ -111,7 +112,9 @@ void CheckCase(std::stringstream& remaining_output, TestCase const& TC, << "\n actual regex string \"" << TC.substituted_regex << "\"" << "\n started matching near: " << first_line; } - if (TC.regex->Match(line)) return; + if (TC.regex->Match(line)) { + return; + } BM_CHECK(TC.match_rule != MR_Next) << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str << "\"" @@ -143,10 +146,10 @@ class TestReporter : public benchmark::BenchmarkReporter { TestReporter(std::vector reps) : reporters_(std::move(reps)) {} - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { + bool ReportContext(const Context& context) override { bool last_ret = false; bool first = true; - for (auto rep : reporters_) { + for (auto* rep : reporters_) { bool new_ret = rep->ReportContext(context); BM_CHECK(first || new_ret == last_ret) << "Reports return different values for ReportContext"; @@ -157,11 +160,15 @@ class TestReporter : public benchmark::BenchmarkReporter { return last_ret; } - void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { - for (auto rep : reporters_) rep->ReportRuns(report); + void ReportRuns(const std::vector& report) override { + for (auto* rep : reporters_) { + rep->ReportRuns(report); + } } - void Finalize() BENCHMARK_OVERRIDE { - for (auto rep : reporters_) rep->Finalize(); + void Finalize() override { + for (auto* rep : reporters_) { + rep->Finalize(); + } } private: @@ -199,15 +206,17 @@ class ResultsChecker { void SetHeader_(const std::string& csv_header); void SetValues_(const std::string& entry_csv_line); - std::vector SplitCsv_(const std::string& line); + std::vector SplitCsv_(const std::string& line) const; }; +namespace { // store the static ResultsChecker in a function to prevent initialization // order problems ResultsChecker& GetResultsChecker() { static ResultsChecker rc; return rc; } +} // end namespace // add a results checker for a benchmark void ResultsChecker::Add(const std::string& entry_pattern, @@ -223,14 +232,16 @@ void ResultsChecker::CheckResults(std::stringstream& output) { // clear before calling tellg() output.clear(); // seek to zero only when needed - if (output.tellg() > start) output.seekg(start); + if (output.tellg() > start) { + output.seekg(start); + } // and just in case output.clear(); } // now go over every line and publish it to the ResultsChecker std::string line; bool on_first = true; - while (output.eof() == false) { + while (!output.eof()) { BM_CHECK(output.good()); std::getline(output, line); if (on_first) { @@ -248,9 +259,8 @@ void ResultsChecker::CheckResults(std::stringstream& output) { if (!p.regex->Match(r.name)) { BM_VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n"; continue; - } else { - BM_VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; } + BM_VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; BM_VLOG(1) << "Checking results of " << r.name << ": ... \n"; p.fn(r); BM_VLOG(1) << "Checking results of " << r.name << ": OK.\n"; @@ -265,7 +275,9 @@ void ResultsChecker::SetHeader_(const std::string& csv_header) { // set the values for a benchmark void ResultsChecker::SetValues_(const std::string& entry_csv_line) { - if (entry_csv_line.empty()) return; // some lines are empty + if (entry_csv_line.empty()) { + return; + } // some lines are empty BM_CHECK(!field_names.empty()); auto vals = SplitCsv_(entry_csv_line); BM_CHECK_EQ(vals.size(), field_names.size()); @@ -277,30 +289,45 @@ void ResultsChecker::SetValues_(const std::string& entry_csv_line) { } // a quick'n'dirty csv splitter (eliminating quotes) -std::vector ResultsChecker::SplitCsv_(const std::string& line) { +std::vector ResultsChecker::SplitCsv_( + const std::string& line) const { std::vector out; - if (line.empty()) return out; - if (!field_names.empty()) out.reserve(field_names.size()); - size_t prev = 0, pos = line.find_first_of(','), curr = pos; - while (pos != line.npos) { + if (line.empty()) { + return out; + } + if (!field_names.empty()) { + out.reserve(field_names.size()); + } + size_t prev = 0; + size_t pos = line.find_first_of(','); + size_t curr = pos; + while (pos != std::string::npos) { BM_CHECK(curr > 0); - if (line[prev] == '"') ++prev; - if (line[curr - 1] == '"') --curr; + if (line[prev] == '"') { + ++prev; + } + if (line[curr - 1] == '"') { + --curr; + } out.push_back(line.substr(prev, curr - prev)); prev = pos + 1; pos = line.find_first_of(',', pos + 1); curr = pos; } curr = line.size(); - if (line[prev] == '"') ++prev; - if (line[curr - 1] == '"') --curr; + if (line[prev] == '"') { + ++prev; + } + if (line[curr - 1] == '"') { + --curr; + } out.push_back(line.substr(prev, curr - prev)); return out; } } // end namespace internal -size_t AddChecker(const char* bm_name, const ResultsCheckFn& fn) { +size_t AddChecker(const std::string& bm_name, const ResultsCheckFn& fn) { auto& rc = internal::GetResultsChecker(); rc.Add(bm_name, fn); return rc.results.size(); @@ -308,7 +335,9 @@ size_t AddChecker(const char* bm_name, const ResultsCheckFn& fn) { int Results::NumThreads() const { auto pos = name.find("/threads:"); - if (pos == name.npos) return 1; + if (pos == std::string::npos) { + return 1; + } auto end = name.find('/', pos + 9); std::stringstream ss; ss << name.substr(pos + 9, end); @@ -324,20 +353,22 @@ double Results::GetTime(BenchmarkTime which) const { BM_CHECK(which == kCpuTime || which == kRealTime); const char* which_str = which == kCpuTime ? "cpu_time" : "real_time"; double val = GetAs(which_str); - auto unit = Get("time_unit"); + const auto* unit = Get("time_unit"); BM_CHECK(unit); if (*unit == "ns") { return val * 1.e-9; - } else if (*unit == "us") { + } + if (*unit == "us") { return val * 1.e-6; - } else if (*unit == "ms") { + } + if (*unit == "ms") { return val * 1.e-3; - } else if (*unit == "s") { + } + if (*unit == "s") { return val; - } else { - BM_CHECK(1 == 0) << "unknown time unit: " << *unit; - return 0; } + BM_CHECK(1 == 0) << "unknown time unit: " << *unit; + return 0; } // ========================================================================= // @@ -376,7 +407,9 @@ int SetSubstitutions( break; } } - if (!exists) subs.push_back(std::move(KV)); + if (!exists) { + subs.push_back(std::move(KV)); + } } return 0; } @@ -393,14 +426,14 @@ void RunOutputTests(int argc, char* argv[]) { benchmark::JSONReporter JR; benchmark::CSVReporter CSVR; struct ReporterTest { - const char* name; + std::string name; std::vector& output_cases; std::vector& error_cases; benchmark::BenchmarkReporter& reporter; std::stringstream out_stream; std::stringstream err_stream; - ReporterTest(const char* n, std::vector& out_tc, + ReporterTest(const std::string& n, std::vector& out_tc, std::vector& err_tc, benchmark::BenchmarkReporter& br) : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) { @@ -408,12 +441,12 @@ void RunOutputTests(int argc, char* argv[]) { reporter.SetErrorStream(&err_stream); } } TestCases[] = { - {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut), + {std::string("ConsoleReporter"), GetTestCaseList(TC_ConsoleOut), GetTestCaseList(TC_ConsoleErr), CR}, - {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr), - JR}, - {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr), - CSVR}, + {std::string("JSONReporter"), GetTestCaseList(TC_JSONOut), + GetTestCaseList(TC_JSONErr), JR}, + {std::string("CSVReporter"), GetTestCaseList(TC_CSVOut), + GetTestCaseList(TC_CSVErr), CSVR}, }; // Create the test reporter and run the benchmarks. @@ -422,7 +455,8 @@ void RunOutputTests(int argc, char* argv[]) { benchmark::RunSpecifiedBenchmarks(&test_rep); for (auto& rep_test : TestCases) { - std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n"; + std::string msg = + std::string("\nTesting ") + rep_test.name + std::string(" Output\n"); std::string banner(msg.size() - 1, '-'); std::cout << banner << msg << banner << "\n"; @@ -439,57 +473,67 @@ void RunOutputTests(int argc, char* argv[]) { // the checks to subscribees. auto& csv = TestCases[2]; // would use == but gcc spits a warning - BM_CHECK(std::strcmp(csv.name, "CSVReporter") == 0); + BM_CHECK(csv.name == std::string("CSVReporter")); internal::GetResultsChecker().CheckResults(csv.out_stream); } BENCHMARK_RESTORE_DEPRECATED_WARNING int SubstrCnt(const std::string& haystack, const std::string& pat) { - if (pat.length() == 0) return 0; + if (pat.length() == 0) { + return 0; + } int count = 0; for (size_t offset = haystack.find(pat); offset != std::string::npos; - offset = haystack.find(pat, offset + pat.length())) + offset = haystack.find(pat, offset + pat.length())) { ++count; + } return count; } -static char ToHex(int ch) { +namespace { +char ToHex(int ch) { return ch < 10 ? static_cast('0' + ch) : static_cast('a' + (ch - 10)); } -static char RandomHexChar() { +char RandomHexChar() { static std::mt19937 rd{std::random_device{}()}; static std::uniform_int_distribution mrand{0, 15}; return ToHex(mrand(rd)); } -static std::string GetRandomFileName() { +std::string GetRandomFileName() { std::string model = "test.%%%%%%"; for (auto& ch : model) { - if (ch == '%') ch = RandomHexChar(); + if (ch == '%') { + ch = RandomHexChar(); + } } return model; } -static bool FileExists(std::string const& name) { +bool FileExists(std::string const& name) { std::ifstream in(name.c_str()); return in.good(); } -static std::string GetTempFileName() { +std::string GetTempFileName() { // This function attempts to avoid race conditions where two tests // create the same file at the same time. However, it still introduces races // similar to tmpnam. int retries = 3; - while (--retries) { + while (--retries != 0) { std::string name = GetRandomFileName(); - if (!FileExists(name)) return name; + if (!FileExists(name)) { + return name; + } } - std::cerr << "Failed to create unique temporary file name" << std::endl; - std::abort(); + std::cerr << "Failed to create unique temporary file name\n"; + std::flush(std::cerr); + std::exit(1); } +} // end namespace std::string GetFileReporterOutput(int argc, char* argv[]) { std::vector new_argv(argv, argv + argc); @@ -502,7 +546,7 @@ std::string GetFileReporterOutput(int argc, char* argv[]) { tmp += tmp_file_name; new_argv.emplace_back(const_cast(tmp.c_str())); - argc = int(new_argv.size()); + argc = static_cast(new_argv.size()); benchmark::Initialize(&argc, new_argv.data()); benchmark::RunSpecifiedBenchmarks(); diff --git a/test/overload_test.cc b/test/overload_test.cc new file mode 100644 index 0000000000..0a62b78605 --- /dev/null +++ b/test/overload_test.cc @@ -0,0 +1,37 @@ +#include "benchmark/macros.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" + +namespace { +// Simulate an overloaded function name. +// This version does nothing and is just here to create ambiguity for +// MyOverloadedBenchmark. +BENCHMARK_UNUSED void MyOverloadedBenchmark() {} + +// This is the actual benchmark function we want to register. +// It has the signature void(benchmark::State&) required by the library. +void MyOverloadedBenchmark(benchmark::State& state) { + for (auto _ : state) { + } +} + +// This macro invocation should compile correctly if benchmark.h +// contains the fix (using static_cast), but would fail to compile +// if the benchmark name were ambiguous (e.g., when using + or no cast +// with an overloaded function). +BENCHMARK(MyOverloadedBenchmark); + +// Also test BENCHMARK_TEMPLATE with an overloaded name. +template +void MyTemplatedOverloadedBenchmark() {} + +template +void MyTemplatedOverloadedBenchmark(benchmark::State& state) { + for (auto _ : state) { + } +} + +BENCHMARK_TEMPLATE(MyTemplatedOverloadedBenchmark, 1); +} // end namespace + +BENCHMARK_MAIN(); diff --git a/test/perf_counters_gtest.cc b/test/perf_counters_gtest.cc index f9e6a6fc96..f3fd10bc95 100644 --- a/test/perf_counters_gtest.cc +++ b/test/perf_counters_gtest.cc @@ -1,6 +1,12 @@ +#include +#include +#include +#include #include +#include #include "../src/perf_counters.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" #ifndef GTEST_SKIP @@ -13,22 +19,42 @@ struct MsgHandler { using benchmark::internal::PerfCounters; using benchmark::internal::PerfCountersMeasurement; using benchmark::internal::PerfCounterValues; +using ::testing::AllOf; +using ::testing::Gt; +using ::testing::Lt; namespace { const char kGenericPerfEvent1[] = "CYCLES"; -const char kGenericPerfEvent2[] = "BRANCHES"; -const char kGenericPerfEvent3[] = "INSTRUCTIONS"; +const char kGenericPerfEvent2[] = "INSTRUCTIONS"; + +std::set UniqueCounterNames(const PerfCounters& counters) { + return {counters.names().begin(), counters.names().end()}; +} + +bool HasRequiredPerfCounters(const std::vector& names) { + if (!PerfCounters::kSupported) { + return false; + } + auto counters = PerfCounters::Create(names); + auto actual_names = UniqueCounterNames(counters); + for (const auto& name : names) { + if (actual_names.find(name) == actual_names.end()) { + return false; + } + } + return true; +} TEST(PerfCountersTest, Init) { EXPECT_EQ(PerfCounters::Initialize(), PerfCounters::kSupported); } TEST(PerfCountersTest, OneCounter) { - if (!PerfCounters::kSupported) { - GTEST_SKIP() << "Performance counters not supported.\n"; + if (!HasRequiredPerfCounters({kGenericPerfEvent1})) { + GTEST_SKIP() << "Requested performance counters are not available."; } - EXPECT_TRUE(PerfCounters::Initialize()); - EXPECT_TRUE(PerfCounters::Create({kGenericPerfEvent1}).IsValid()); + auto counter = PerfCounters::Create({kGenericPerfEvent1}); + EXPECT_EQ(UniqueCounterNames(counter).size(), 1); } TEST(PerfCountersTest, NegativeTest) { @@ -36,158 +62,340 @@ TEST(PerfCountersTest, NegativeTest) { EXPECT_FALSE(PerfCounters::Initialize()); return; } - EXPECT_TRUE(PerfCounters::Initialize()); - EXPECT_FALSE(PerfCounters::Create({}).IsValid()); - EXPECT_FALSE(PerfCounters::Create({""}).IsValid()); - EXPECT_FALSE(PerfCounters::Create({"not a counter name"}).IsValid()); + if (!HasRequiredPerfCounters({kGenericPerfEvent2, kGenericPerfEvent1})) { + GTEST_SKIP() << "Requested performance counters are not available."; + } + // Safety checks + // Create() will always create a valid object, even if passed no or + // wrong arguments as the new behavior is to warn and drop unsupported + // counters + EXPECT_EQ(PerfCounters::Create({}).num_counters(), 0); + EXPECT_EQ(PerfCounters::Create({""}).num_counters(), 0); + EXPECT_EQ(PerfCounters::Create({"not a counter name"}).num_counters(), 0); { - EXPECT_TRUE(PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2, - kGenericPerfEvent3}) - .IsValid()); - } - EXPECT_FALSE( - PerfCounters::Create({kGenericPerfEvent2, "", kGenericPerfEvent1}) - .IsValid()); - EXPECT_FALSE(PerfCounters::Create({kGenericPerfEvent3, "not a counter name", - kGenericPerfEvent1}) - .IsValid()); + // Try sneaking in a bad egg to see if it is filtered out. The + // number of counters has to be two, not zero + auto counter = + PerfCounters::Create({kGenericPerfEvent2, "", kGenericPerfEvent1}); + auto names = UniqueCounterNames(counter); + EXPECT_EQ(names.size(), 2); + EXPECT_EQ(names, + std::set({kGenericPerfEvent2, kGenericPerfEvent1})); + } { - EXPECT_TRUE(PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2, - kGenericPerfEvent3}) - .IsValid()); - } - EXPECT_FALSE( - PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2, - kGenericPerfEvent3, "MISPREDICTED_BRANCH_RETIRED"}) - .IsValid()); + // Try sneaking in an outrageous counter, like a fat finger mistake + auto counter = PerfCounters::Create( + {kGenericPerfEvent2, "not a counter name", kGenericPerfEvent1}); + auto names = UniqueCounterNames(counter); + EXPECT_EQ(names.size(), 2); + EXPECT_EQ(names, + std::set({kGenericPerfEvent2, kGenericPerfEvent1})); + } + { + // Finally try a golden input - it should like both of them + EXPECT_EQ(UniqueCounterNames(PerfCounters::Create( + {kGenericPerfEvent1, kGenericPerfEvent2})) + .size(), + 2); + } + { + // Add a bad apple in the end of the chain to check the edges + auto counter = PerfCounters::Create( + {kGenericPerfEvent1, kGenericPerfEvent2, "bad event name"}); + auto names = UniqueCounterNames(counter); + EXPECT_EQ(names.size(), 2); + EXPECT_EQ(names, + std::set({kGenericPerfEvent1, kGenericPerfEvent2})); + } +} + +static std::map SnapshotAndCombine( + PerfCounters& counters) { + PerfCounterValues values(counters.num_counters()); + std::map value_map; + + if (counters.Snapshot(&values)) { + for (size_t i = 0; i != counters.num_counters(); ++i) { + value_map[counters.names()[i]] += values[i]; + } + } + return value_map; } TEST(PerfCountersTest, Read1Counter) { - if (!PerfCounters::kSupported) { - GTEST_SKIP() << "Test skipped because libpfm is not supported.\n"; + if (!HasRequiredPerfCounters({kGenericPerfEvent1})) { + GTEST_SKIP() << "Requested performance counters are not available."; } - EXPECT_TRUE(PerfCounters::Initialize()); auto counters = PerfCounters::Create({kGenericPerfEvent1}); - EXPECT_TRUE(counters.IsValid()); - PerfCounterValues values1(1); - EXPECT_TRUE(counters.Snapshot(&values1)); - EXPECT_GT(values1[0], 0); - PerfCounterValues values2(1); - EXPECT_TRUE(counters.Snapshot(&values2)); - EXPECT_GT(values2[0], 0); - EXPECT_GT(values2[0], values1[0]); + auto values1 = SnapshotAndCombine(counters); + EXPECT_EQ(values1.size(), 1); + EXPECT_GT(values1.begin()->second, 0); + auto values2 = SnapshotAndCombine(counters); + EXPECT_EQ(values2.size(), 1); + EXPECT_GT(values2.begin()->second, 0); + EXPECT_GT(values2.begin()->second, values1.begin()->second); +} + +TEST(PerfCountersTest, Read1CounterEachCPU) { + if (!HasRequiredPerfCounters({kGenericPerfEvent1})) { + GTEST_SKIP() << "Requested performance counters are not available."; + } +#ifdef __linux__ + cpu_set_t saved_set; + if (sched_getaffinity(0, sizeof(saved_set), &saved_set) != 0) { + // This can happen e.g. if there are more than CPU_SETSIZE CPUs. + GTEST_SKIP() << "Could not save CPU affinity mask."; + } + + for (size_t cpu = 0; cpu != CPU_SETSIZE; ++cpu) { + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + if (sched_setaffinity(0, sizeof(set), &set) != 0) { + break; + } + + auto counters = PerfCounters::Create({kGenericPerfEvent1}); + auto values1 = SnapshotAndCombine(counters); + EXPECT_EQ(values1.size(), 1); + EXPECT_GT(values1.begin()->second, 0); + auto values2 = SnapshotAndCombine(counters); + EXPECT_EQ(values2.size(), 1); + EXPECT_GT(values2.begin()->second, 0); + EXPECT_GT(values2.begin()->second, values1.begin()->second); + } + + EXPECT_EQ(sched_setaffinity(0, sizeof(saved_set), &saved_set), 0); +#else + GTEST_SKIP() << "Test skipped on non-Linux."; +#endif } TEST(PerfCountersTest, Read2Counters) { - if (!PerfCounters::kSupported) { - GTEST_SKIP() << "Test skipped because libpfm is not supported.\n"; + if (!HasRequiredPerfCounters({kGenericPerfEvent1, kGenericPerfEvent2})) { + GTEST_SKIP() << "Requested performance counters are not available."; } - EXPECT_TRUE(PerfCounters::Initialize()); auto counters = PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2}); - EXPECT_TRUE(counters.IsValid()); - PerfCounterValues values1(2); - EXPECT_TRUE(counters.Snapshot(&values1)); - EXPECT_GT(values1[0], 0); - EXPECT_GT(values1[1], 0); - PerfCounterValues values2(2); - EXPECT_TRUE(counters.Snapshot(&values2)); - EXPECT_GT(values2[0], 0); - EXPECT_GT(values2[1], 0); + auto values1 = SnapshotAndCombine(counters); + EXPECT_EQ(values1.size(), 2); + for (auto& kv : values1) { + EXPECT_GT(kv.second, 0); + } + auto values2 = SnapshotAndCombine(counters); + EXPECT_EQ(values1.size(), 2); + for (auto& kv : values2) { + EXPECT_GT(kv.second, 0); + EXPECT_GT(kv.second, values1[kv.first]); + } } TEST(PerfCountersTest, ReopenExistingCounters) { - // The test works (i.e. causes read to fail) for the assumptions - // about hardware capabilities (i.e. small number (3-4) hardware - // counters) at this date. - if (!PerfCounters::kSupported) { - GTEST_SKIP() << "Test skipped because libpfm is not supported.\n"; + // This test works in recent and old Intel hardware, Pixel 3, and Pixel 6. + // However we cannot make assumptions beyond 2 HW counters due to Pixel 6. + if (!HasRequiredPerfCounters({kGenericPerfEvent1})) { + GTEST_SKIP() << "Requested performance counters are not available."; } - EXPECT_TRUE(PerfCounters::Initialize()); - std::vector counters; - counters.reserve(6); - for (int i = 0; i < 6; i++) - counters.push_back(PerfCounters::Create({kGenericPerfEvent1})); - PerfCounterValues values(1); + std::vector kMetrics({kGenericPerfEvent1}); + std::vector counters(2); + for (auto& counter : counters) { + counter = PerfCounters::Create(kMetrics); + } + PerfCounterValues values(counters[0].num_counters()); EXPECT_TRUE(counters[0].Snapshot(&values)); - EXPECT_FALSE(counters[4].Snapshot(&values)); - EXPECT_FALSE(counters[5].Snapshot(&values)); + EXPECT_TRUE(counters[1].Snapshot(&values)); } TEST(PerfCountersTest, CreateExistingMeasurements) { // The test works (i.e. causes read to fail) for the assumptions - // about hardware capabilities (i.e. small number (3-4) hardware + // about hardware capabilities (i.e. small number (2) hardware // counters) at this date, // the same as previous test ReopenExistingCounters. if (!PerfCounters::kSupported) { - GTEST_SKIP() << "Test skipped because libpfm is not supported.\n"; + GTEST_SKIP() << "Test skipped because libpfm is not supported."; + } + + // This means we will try 10 counters but we can only guarantee + // for sure at this time that only 3 will work. Perhaps in the future + // we could use libpfm to query for the hardware limits on this + // particular platform. + const int kMaxCounters = 10; + const int kMinValidCounters = 2; + + // Let's use a ubiquitous counter that is guaranteed to work + // on all platforms + const std::vector kMetrics{"cycles"}; + if (!HasRequiredPerfCounters(kMetrics)) { + GTEST_SKIP() << "Requested performance counters are not available."; } - EXPECT_TRUE(PerfCounters::Initialize()); - std::vector perf_counter_measurements; - std::vector> measurements; - perf_counter_measurements.reserve(10); - for (int i = 0; i < 10; i++) + // Cannot create a vector of actual objects because the + // copy constructor of PerfCounters is deleted - and so is + // implicitly deleted on PerfCountersMeasurement too + std::vector> + perf_counter_measurements; + + perf_counter_measurements.reserve(kMaxCounters); + for (int j = 0; j < kMaxCounters; ++j) { perf_counter_measurements.emplace_back( - std::vector{kGenericPerfEvent1}); + new PerfCountersMeasurement(kMetrics)); + } - perf_counter_measurements[0].Start(); - EXPECT_TRUE(perf_counter_measurements[0].Stop(measurements)); + std::vector> measurements; + + // Start all counters together to see if they hold + size_t max_counters = kMaxCounters; + for (size_t i = 0; i < kMaxCounters; ++i) { + auto& counter(*perf_counter_measurements[i]); + std::set names{counter.names().begin(), counter.names().end()}; + EXPECT_EQ(names.size(), 1); + if (!counter.Start()) { + max_counters = i; + break; + }; + } - measurements.clear(); - perf_counter_measurements[8].Start(); - EXPECT_FALSE(perf_counter_measurements[8].Stop(measurements)); + ASSERT_GE(max_counters, kMinValidCounters); - measurements.clear(); - perf_counter_measurements[9].Start(); - EXPECT_FALSE(perf_counter_measurements[9].Stop(measurements)); + // Start all together + for (size_t i = 0; i < max_counters; ++i) { + auto& counter(*perf_counter_measurements[i]); + EXPECT_TRUE(counter.Stop(measurements) || (i >= kMinValidCounters)); + } + + // Start/stop individually + for (size_t i = 0; i < max_counters; ++i) { + auto& counter(*perf_counter_measurements[i]); + measurements.clear(); + counter.Start(); + EXPECT_TRUE(counter.Stop(measurements) || (i >= kMinValidCounters)); + } } -size_t do_work() { - size_t res = 0; - for (size_t i = 0; i < 100000000; ++i) res += i * i; - return res; +// We try to do some meaningful work here but the compiler +// insists in optimizing away our loop so we had to add a +// no-optimize macro. In case it fails, we added some entropy +// to this pool as well. + +BENCHMARK_DONT_OPTIMIZE size_t do_work() { + static std::mt19937 rd{std::random_device{}()}; + static std::uniform_int_distribution mrand(0, 10); + const size_t kNumLoops = 1000000; + size_t sum = 0; + for (size_t j = 0; j < kNumLoops; ++j) { + sum += mrand(rd); + } + benchmark::DoNotOptimize(sum); + return sum; } -void measure(size_t threadcount, PerfCounterValues* values1, - PerfCounterValues* values2) { - BM_CHECK_NE(values1, nullptr); - BM_CHECK_NE(values2, nullptr); +void measure(size_t threadcount, std::map* before, + std::map* after) { + BM_CHECK_NE(before, nullptr); + BM_CHECK_NE(after, nullptr); std::vector threads(threadcount); - auto work = [&]() { BM_CHECK(do_work() > 1000); }; + // Because we do not care whether the threads execute concurrently, but we do + // care that they do all of their work between the SnapshotAndCombine calls, + // we serialize them with a mutex. See + // https://github.com/google/benchmark/issues/2173. + std::mutex mutex; + auto work = [&mutex]() { + mutex.lock(); + BM_CHECK(do_work() > 1000); + mutex.unlock(); + }; // We need to first set up the counters, then start the threads, so the - // threads would inherit the counters. But later, we need to first destroy the - // thread pool (so all the work finishes), then measure the counters. So the - // scopes overlap, and we need to explicitly control the scope of the + // threads would inherit the counters. But later, we need to first destroy + // the thread pool (so all the work finishes), then measure the counters. So + // the scopes overlap, and we need to explicitly control the scope of the // threadpool. auto counters = - PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent3}); - for (auto& t : threads) t = std::thread(work); - counters.Snapshot(values1); - for (auto& t : threads) t.join(); - counters.Snapshot(values2); + PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2}); + mutex.lock(); + for (auto& t : threads) { + t = std::thread(work); + } + *before = SnapshotAndCombine(counters); + mutex.unlock(); + for (auto& t : threads) { + t.join(); + } + *after = SnapshotAndCombine(counters); } TEST(PerfCountersTest, MultiThreaded) { + if (!HasRequiredPerfCounters({kGenericPerfEvent1, kGenericPerfEvent2})) { + GTEST_SKIP() << "Requested performance counters are not available."; + } + std::map before, after; + + // Notice that this test will work even if we taskset it to a single CPU + // In this case the threads will run sequentially + // Start two threads and measure the number of combined cycles and + // instructions + measure(2, &before, &after); + std::vector Elapsed2Threads{ + static_cast(after[kGenericPerfEvent1] - + before[kGenericPerfEvent1]), + static_cast(after[kGenericPerfEvent2] - + before[kGenericPerfEvent2])}; + + // Start four threads and measure the number of combined cycles and + // instructions + measure(4, &before, &after); + std::vector Elapsed4Threads{ + static_cast(after[kGenericPerfEvent1] - + before[kGenericPerfEvent1]), + static_cast(after[kGenericPerfEvent2] - + before[kGenericPerfEvent2])}; + + // The following expectations fail (at least on a beefy workstation with lots + // of cpus) - it seems that in some circumstances the runtime of 4 threads + // can even be better than with 2. + // So instead of expecting 4 threads to be slower, let's just make sure they + // do not differ too much in general (one is not more than 10x than the + // other). + EXPECT_THAT(Elapsed4Threads[0] / Elapsed2Threads[0], AllOf(Gt(0.1), Lt(10))); + EXPECT_THAT(Elapsed4Threads[1] / Elapsed2Threads[1], AllOf(Gt(0.1), Lt(10))); +} + +TEST(PerfCountersTest, HardwareLimits) { + // The test works (i.e. causes read to fail) for the assumptions + // about hardware capabilities (i.e. small number (3-4) hardware + // counters) at this date, + // the same as previous test ReopenExistingCounters. if (!PerfCounters::kSupported) { GTEST_SKIP() << "Test skipped because libpfm is not supported."; } EXPECT_TRUE(PerfCounters::Initialize()); - PerfCounterValues values1(2); - PerfCounterValues values2(2); - measure(2, &values1, &values2); - std::vector D1{static_cast(values2[0] - values1[0]), - static_cast(values2[1] - values1[1])}; + // Taken from `perf list`, but focusses only on those HW events that actually + // were reported when running `sudo perf stat -a sleep 10`, intersected over + // several platforms. All HW events listed in the first command not reported + // in the second seem to not work. This is sad as we don't really get to test + // the grouping here (groups can contain up to 6 members)... + std::vector counter_names{ + "cycles", // leader + "instructions", // + "branch-misses", // + }; - measure(4, &values1, &values2); - std::vector D2{static_cast(values2[0] - values1[0]), - static_cast(values2[1] - values1[1])}; + // In the off-chance that some of these values are not supported, + // we filter them out so the test will complete without failure + // albeit it might not actually test the grouping on that platform + std::vector valid_names; + for (const std::string& name : counter_names) { + if (PerfCounters::IsCounterSupported(name)) { + valid_names.push_back(name); + } + } + PerfCountersMeasurement counter(valid_names); + + std::vector> measurements; - // Some extra work will happen on the main thread - like joining the threads - // - so the ratio won't be quite 2.0, but very close. - EXPECT_GE(D2[0], 1.9 * D1[0]); - EXPECT_GE(D2[1], 1.9 * D1[1]); + counter.Start(); + EXPECT_TRUE(counter.Stop(measurements)); } + } // namespace diff --git a/test/perf_counters_test.cc b/test/perf_counters_test.cc index 3017a452fe..d88f6bfd5b 100644 --- a/test/perf_counters_test.cc +++ b/test/perf_counters_test.cc @@ -1,27 +1,123 @@ +#include +#include +#include +#include #undef NDEBUG +#include "../src/commandlineflags.h" #include "../src/perf_counters.h" - -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/utils.h" #include "output_test.h" -static void BM_Simple(benchmark::State& state) { +namespace benchmark { + +BM_DECLARE_string(benchmark_perf_counters); + +} // namespace benchmark +namespace { +const char kGenericPerfEvent1[] = "CYCLES"; +const char kGenericPerfEvent2[] = "INSTRUCTIONS"; + +std::set UniqueCounterNames( + const benchmark::internal::PerfCounters& counters) { + return {counters.names().begin(), counters.names().end()}; +} + +bool HasRequiredPerfCounters(const std::vector& names) { + if (!benchmark::internal::PerfCounters::kSupported) { + return false; + } + auto counters = benchmark::internal::PerfCounters::Create(names); + auto actual_names = UniqueCounterNames(counters); + for (const auto& name : names) { + if (actual_names.find(name) == actual_names.end()) { + return false; + } + } + return true; +} + +void BM_Simple(benchmark::State& state) { for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } } BENCHMARK(BM_Simple); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Simple\",$"}}); +const int kIters = 1000000; + +void BM_WithoutPauseResume(benchmark::State& state) { + int n = 0; + + for (auto _ : state) { + for (auto i = 0; i < kIters; ++i) { + n = 1 - n; + benchmark::DoNotOptimize(n); + } + } +} + +BENCHMARK(BM_WithoutPauseResume); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_WithoutPauseResume\",$"}}); + +void BM_WithPauseResume(benchmark::State& state) { + int m = 0, n = 0; + + for (auto _ : state) { + for (auto i = 0; i < kIters; ++i) { + n = 1 - n; + benchmark::DoNotOptimize(n); + } + + state.PauseTiming(); + for (auto j = 0; j < kIters; ++j) { + m = 1 - m; + benchmark::DoNotOptimize(m); + } + state.ResumeTiming(); + } +} + +BENCHMARK(BM_WithPauseResume); + +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_WithPauseResume\",$"}}); + static void CheckSimple(Results const& e) { - CHECK_COUNTER_VALUE(e, double, "CYCLES", GT, 0); - CHECK_COUNTER_VALUE(e, double, "BRANCHES", GT, 0.0); + CHECK_COUNTER_VALUE(e, double, kGenericPerfEvent1, GT, 0); } + +double withoutPauseResumeInstrCount = 0.0; +double withPauseResumeInstrCount = 0.0; + +void SaveInstrCountWithoutResume(Results const& e) { + withoutPauseResumeInstrCount = e.GetAs(kGenericPerfEvent2); +} + +void SaveInstrCountWithResume(Results const& e) { + withPauseResumeInstrCount = e.GetAs(kGenericPerfEvent2); +} + CHECK_BENCHMARK_RESULTS("BM_Simple", &CheckSimple); +CHECK_BENCHMARK_RESULTS("BM_WithoutPauseResume", &SaveInstrCountWithoutResume); +CHECK_BENCHMARK_RESULTS("BM_WithPauseResume", &SaveInstrCountWithResume); +} // end namespace int main(int argc, char* argv[]) { - if (!benchmark::internal::PerfCounters::kSupported) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + if (!HasRequiredPerfCounters({kGenericPerfEvent1, kGenericPerfEvent2})) { return 0; } + benchmark::FLAGS_benchmark_perf_counters = + std::string(kGenericPerfEvent1) + "," + kGenericPerfEvent2; + benchmark::internal::PerfCounters::Initialize(); RunOutputTests(argc, argv); + + BM_CHECK_GT(withPauseResumeInstrCount, kIters); + BM_CHECK_GT(withoutPauseResumeInstrCount, kIters); + BM_CHECK_LT(withPauseResumeInstrCount, 1.5 * withoutPauseResumeInstrCount); } diff --git a/test/profiler_manager_gtest.cc b/test/profiler_manager_gtest.cc new file mode 100644 index 0000000000..6e83e32a90 --- /dev/null +++ b/test/profiler_manager_gtest.cc @@ -0,0 +1,45 @@ +#include + +#include "benchmark/managers.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/utils.h" +#include "gtest/gtest.h" + +namespace { + +class TestProfilerManager : public benchmark::ProfilerManager { + public: + void AfterSetupStart() override { ++start_called; } + void BeforeTeardownStop() override { ++stop_called; } + + int start_called = 0; + int stop_called = 0; +}; + +void BM_empty(benchmark::State& state) { + for (auto _ : state) { + auto iterations = state.iterations(); + benchmark::DoNotOptimize(iterations); + } +} +BENCHMARK(BM_empty); + +TEST(ProfilerManager, ReregisterManager) { +#if GTEST_HAS_DEATH_TEST + // Tests only runnable in debug mode (when BM_CHECK is enabled). +#ifndef NDEBUG +#ifndef TEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS + ASSERT_DEATH_IF_SUPPORTED( + { + std::unique_ptr pm(new TestProfilerManager()); + benchmark::RegisterProfilerManager(pm.get()); + benchmark::RegisterProfilerManager(pm.get()); + }, + "RegisterProfilerManager"); +#endif +#endif +#endif +} + +} // namespace diff --git a/test/profiler_manager_iterations_test.cc b/test/profiler_manager_iterations_test.cc new file mode 100644 index 0000000000..90b34a1daa --- /dev/null +++ b/test/profiler_manager_iterations_test.cc @@ -0,0 +1,66 @@ +#include +#include +#include +#include + +#include "benchmark/benchmark_api.h" +#include "benchmark/managers.h" +#include "benchmark/registration.h" +#include "benchmark/reporter.h" +#include "benchmark/state.h" + +// Tests that we can specify the number of profiler iterations with +// --benchmark_min_time=x. +namespace { + +int iteration_count = 0; +int end_profiler_iteration_count = 0; + +class TestProfilerManager : public benchmark::ProfilerManager { + void AfterSetupStart() override { iteration_count = 0; } + void BeforeTeardownStop() override { + end_profiler_iteration_count = iteration_count; + } +}; + +class NullReporter : public benchmark::BenchmarkReporter { + public: + bool ReportContext(const Context& /*context*/) override { return true; } + void ReportRuns(const std::vector& /* report */) override {} +}; + +void BM_MyBench(benchmark::State& state) { + for (auto s : state) { + ++iteration_count; + } +} +BENCHMARK(BM_MyBench); +} // end namespace + +int main(int argc, char** argv) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + // Make a fake argv and append the new --benchmark_profiler_iterations= + // to it. + int fake_argc = argc + 1; + std::vector fake_argv(static_cast(fake_argc)); + for (size_t i = 0; i < static_cast(argc); ++i) { + fake_argv[i] = argv[i]; + } + fake_argv[static_cast(argc)] = "--benchmark_min_time=4x"; + + std::unique_ptr pm(new TestProfilerManager()); + benchmark::RegisterProfilerManager(pm.get()); + + benchmark::Initialize(&fake_argc, const_cast(fake_argv.data())); + + NullReporter null_reporter; + const size_t returned_count = + benchmark::RunSpecifiedBenchmarks(&null_reporter, "BM_MyBench"); + assert(returned_count == 1); + + // Check the executed iters. + assert(end_profiler_iteration_count == 4); + + benchmark::RegisterProfilerManager(nullptr); + return 0; +} diff --git a/test/profiler_manager_test.cc b/test/profiler_manager_test.cc new file mode 100644 index 0000000000..bd86d42c04 --- /dev/null +++ b/test/profiler_manager_test.cc @@ -0,0 +1,58 @@ +// FIXME: WIP + +#include +#include + +#include "benchmark/benchmark_api.h" +#include "benchmark/managers.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/utils.h" +#include "output_test.h" + +namespace { +class TestProfilerManager : public benchmark::ProfilerManager { + public: + void AfterSetupStart() override { ++start_called; } + void BeforeTeardownStop() override { ++stop_called; } + + int start_called = 0; + int stop_called = 0; +}; + +void BM_empty(benchmark::State& state) { + for (auto _ : state) { + auto iterations = state.iterations(); + benchmark::DoNotOptimize(iterations); + } +} +BENCHMARK(BM_empty); + +ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, + {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_empty\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_empty\",%csv_report$"}}); +} // end namespace + +int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + std::unique_ptr pm(new TestProfilerManager()); + + benchmark::RegisterProfilerManager(pm.get()); + RunOutputTests(argc, argv); + benchmark::RegisterProfilerManager(nullptr); + + assert(pm->start_called == 1); + assert(pm->stop_called == 1); +} diff --git a/test/register_benchmark_test.cc b/test/register_benchmark_test.cc index 37dbba6b95..c662d494b4 100644 --- a/test/register_benchmark_test.cc +++ b/test/register_benchmark_test.cc @@ -4,13 +4,16 @@ #include #include "../src/check.h" // NOTE: check.h is for internal use only! -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/reporter.h" +#include "benchmark/state.h" namespace { class TestReporter : public benchmark::ConsoleReporter { public: - virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + void ReportRuns(const std::vector& report) override { all_runs_.insert(all_runs_.end(), begin(report), end(report)); ConsoleReporter::ReportRuns(report); } @@ -19,11 +22,11 @@ class TestReporter : public benchmark::ConsoleReporter { }; struct TestCase { - std::string name; - const char* label; + const std::string name; + const std::string label; // Note: not explicit as we rely on it being converted through ADD_CASES. - TestCase(const char* xname) : TestCase(xname, nullptr) {} - TestCase(const char* xname, const char* xlabel) + TestCase(const std::string& xname) : TestCase(xname, "") {} + TestCase(const std::string& xname, const std::string& xlabel) : name(xname), label(xlabel) {} typedef benchmark::BenchmarkReporter::Run Run; @@ -32,7 +35,7 @@ struct TestCase { // clang-format off BM_CHECK(name == run.benchmark_name()) << "expected " << name << " got " << run.benchmark_name(); - if (label) { + if (!label.empty()) { BM_CHECK(run.report_label == label) << "expected " << label << " got " << run.report_label; } else { @@ -53,11 +56,10 @@ int AddCases(std::initializer_list const& v) { #define CONCAT(x, y) CONCAT2(x, y) #define CONCAT2(x, y) x##y -#define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases({__VA_ARGS__}) +#define ADD_CASES(...) \ + const int CONCAT(dummy, __LINE__) = AddCases({__VA_ARGS__}) -} // end namespace - -typedef benchmark::internal::Benchmark* ReturnVal; +using ReturnVal = benchmark::Benchmark const* const; //----------------------------------------------------------------------------// // Test RegisterBenchmark with no additional arguments @@ -76,7 +78,6 @@ ADD_CASES({"BM_function"}, {"BM_function_manual_registration"}); // Note: GCC <= 4.8 do not support this form of RegisterBenchmark because they // reject the variadic pack expansion of lambda captures. //----------------------------------------------------------------------------// -#ifndef BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK void BM_extra_args(benchmark::State& st, const char* label) { for (auto _ : st) { @@ -86,15 +87,14 @@ void BM_extra_args(benchmark::State& st, const char* label) { int RegisterFromFunction() { std::pair cases[] = { {"test1", "One"}, {"test2", "Two"}, {"test3", "Three"}}; - for (auto const& c : cases) + for (auto const& c : cases) { benchmark::RegisterBenchmark(c.first, &BM_extra_args, c.second); + } return 0; } -int dummy2 = RegisterFromFunction(); +const int dummy2 = RegisterFromFunction(); ADD_CASES({"test1", "One"}, {"test2", "Two"}, {"test3", "Three"}); -#endif // BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK - //----------------------------------------------------------------------------// // Test RegisterBenchmark with DISABLED_ benchmark //----------------------------------------------------------------------------// @@ -107,6 +107,20 @@ ReturnVal dummy3 = benchmark::RegisterBenchmark("DISABLED_BM_function_manual", DISABLED_BM_function); // No need to add cases because we don't expect them to run. +//----------------------------------------------------------------------------// +// Test BENCHMARK_NAMED: verifies name format "func/test_case_name" and that +// chaining (e.g. ->Threads()) works, without introducing a lambda. +//----------------------------------------------------------------------------// +void BM_named(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK_NAMED(BM_named, variant_a); +BENCHMARK_NAMED(BM_named, variant_b); +BENCHMARK_NAMED(BM_named, variant_c)->Threads(2); +ADD_CASES({"BM_named/variant_a"}, {"BM_named/variant_b"}, + {"BM_named/variant_c/threads:2"}); + //----------------------------------------------------------------------------// // Test RegisterBenchmark with different callable types //----------------------------------------------------------------------------// @@ -119,14 +133,11 @@ struct CustomFixture { }; void TestRegistrationAtRuntime() { -#ifdef BENCHMARK_HAS_CXX11 { CustomFixture fx; benchmark::RegisterBenchmark("custom_fixture", fx); - AddCases({"custom_fixture"}); + AddCases({std::string("custom_fixture")}); } -#endif -#ifndef BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK { const char* x = "42"; auto capturing_lam = [=](benchmark::State& st) { @@ -137,7 +148,6 @@ void TestRegistrationAtRuntime() { benchmark::RegisterBenchmark("lambda_benchmark", capturing_lam); AddCases({{"lambda_benchmark", x}}); } -#endif } // Test that all benchmarks, registered at either during static init or runtime, @@ -163,7 +173,7 @@ void RunTestOne() { // benchmarks. // Also test that new benchmarks can be registered and ran afterwards. void RunTestTwo() { - assert(ExpectedResults.size() != 0 && + assert(!ExpectedResults.empty() && "must have at least one registered benchmark"); ExpectedResults.clear(); benchmark::ClearRegisteredBenchmarks(); @@ -187,8 +197,10 @@ void RunTestTwo() { } assert(EB == ExpectedResults.end()); } +} // end namespace int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); benchmark::Initialize(&argc, argv); RunTestOne(); diff --git a/test/repetitions_test.cc b/test/repetitions_test.cc index 569777d5f9..0673692b64 100644 --- a/test/repetitions_test.cc +++ b/test/repetitions_test.cc @@ -1,12 +1,15 @@ -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" #include "output_test.h" +namespace { // ========================================================================= // // ------------------------ Testing Basic Output --------------------------- // // ========================================================================= // -static void BM_ExplicitRepetitions(benchmark::State& state) { +void BM_ExplicitRepetitions(benchmark::State& state) { for (auto _ : state) { } } @@ -108,7 +111,7 @@ ADD_CASES(TC_CSVOut, // ------------------------ Testing Basic Output --------------------------- // // ========================================================================= // -static void BM_ImplicitRepetitions(benchmark::State& state) { +void BM_ImplicitRepetitions(benchmark::State& state) { for (auto _ : state) { } } @@ -207,8 +210,49 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions_mean\",%csv_report$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions_median\",%csv_report$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions_stddev\",%csv_report$"}}); +// ========================================================================= // +// --------------------- Testing Skipped Repetitions ------------------------ // +// ========================================================================= // + +void BM_SkippedFirstRepetition(benchmark::State& state) { + static int repetition_index = 0; + if (repetition_index++ == 0) { + state.SkipWithError("skipped first repetition"); + return; + } + for (auto _ : state) { + } +} +BENCHMARK(BM_SkippedFirstRepetition)->Repetitions(3); + +ADD_CASES( + TC_ConsoleOut, + {{"^BM_SkippedFirstRepetition/repeats:3[ ]+ERROR OCCURRED: " + "'skipped first repetition'$"}, + {"^BM_SkippedFirstRepetition/repeats:3 %console_report$", MR_Next}, + {"^BM_SkippedFirstRepetition/repeats:3 %console_report$", MR_Next}, + {"^BM_SkippedFirstRepetition/repeats:3_mean %console_report$", MR_Next}, + {"^BM_SkippedFirstRepetition/repeats:3_median %console_report$", MR_Next}, + {"^BM_SkippedFirstRepetition/repeats:3_stddev %console_report$", + MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_SkippedFirstRepetition/repeats:3_mean\",$"}, + {"\"run_type\": \"aggregate\",$"}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"aggregate_unit\": \"time\",$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_SkippedFirstRepetition/repeats:3_mean\",2,%float,%float," + "ns,,,,,$"}}); +} // end namespace + // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // // ========================================================================= // -int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } +int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + RunOutputTests(argc, argv); +} diff --git a/test/report_aggregates_only_test.cc b/test/report_aggregates_only_test.cc index 47da503588..cb0ad09b04 100644 --- a/test/report_aggregates_only_test.cc +++ b/test/report_aggregates_only_test.cc @@ -3,9 +3,12 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" #include "output_test.h" +namespace { // Ok this test is super ugly. We want to check what happens with the file // reporter in the presence of ReportAggregatesOnly(). // We do not care about console output, the normal tests check that already. @@ -15,8 +18,10 @@ void BM_SummaryRepeat(benchmark::State& state) { } } BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly(); +} // end namespace int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); const std::string output = GetFileReporterOutput(argc, argv); if (SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3") != 4 || diff --git a/test/reporter_list_gtest.cc b/test/reporter_list_gtest.cc new file mode 100644 index 0000000000..74d5d02dd6 --- /dev/null +++ b/test/reporter_list_gtest.cc @@ -0,0 +1,87 @@ +#include +#include +#include + +#include "../src/benchmark_api_internal.h" +#include "benchmark/benchmark.h" +#include "gtest/gtest.h" + +namespace benchmark { +namespace internal { +namespace { + +void BM_ReporterListDummy(::benchmark::State& state) { + for (auto _ : state) { + } +} + +// Registers two benchmarks (once) and returns the matching instances, the +// same way RunSpecifiedBenchmarks() selects them for --benchmark_list_tests. +const std::vector& ListTestBenchmarks() { + static const std::vector* const benchmarks = [] { + RegisterBenchmark("BM_ReporterListFirst", BM_ReporterListDummy); + RegisterBenchmark("BM_ReporterListSecond", BM_ReporterListDummy); + auto* result = new std::vector(); + std::ostringstream err_stream; + FindBenchmarksInternal("BM_ReporterList.*", result, &err_stream); + return result; + }(); + return *benchmarks; +} + +template +std::string ListedOutput() { + Reporter reporter; + std::ostringstream out; + reporter.SetOutputStream(&out); + reporter.List(ListTestBenchmarks()); + return out.str(); +} + +TEST(ReporterListTest, FindsRegisteredBenchmarks) { + ASSERT_EQ(ListTestBenchmarks().size(), 2u); +} + +TEST(ReporterListTest, DefaultListsOneNamePerLine) { + EXPECT_EQ(ListedOutput(), + "BM_ReporterListFirst\nBM_ReporterListSecond\n"); +} + +TEST(ReporterListTest, JSONListsNamesInBenchmarksArray) { + EXPECT_EQ(ListedOutput(), + "{\n" + " \"benchmarks\": [\n" + " {\n" + " \"name\": \"BM_ReporterListFirst\"\n" + " },\n" + " {\n" + " \"name\": \"BM_ReporterListSecond\"\n" + " }\n" + " ]\n" + "}\n"); +} + +TEST(ReporterListTest, JSONListsNoBenchmarks) { + JSONReporter reporter; + std::ostringstream out; + reporter.SetOutputStream(&out); + reporter.List({}); + EXPECT_EQ(out.str(), + "{\n" + " \"benchmarks\": [\n" + " ]\n" + "}\n"); +} + +TEST(ReporterListTest, CSVListsNameColumn) { + BENCHMARK_DISABLE_DEPRECATED_WARNING + EXPECT_EQ(ListedOutput(), + "name\n" + "\"BM_ReporterListFirst\"\n" + "\"BM_ReporterListSecond\"\n"); + BENCHMARK_RESTORE_DEPRECATED_WARNING +} + +} // namespace +} // namespace internal +} // namespace benchmark diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 65bb14a171..26d87de012 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -1,11 +1,15 @@ - #undef NDEBUG -#include -#include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/statistics.h" +#include "benchmark/sysinfo.h" +#include "benchmark/types.h" +#include "benchmark/utils.h" #include "output_test.h" +namespace { // ========================================================================= // // ---------------------- Testing Prologue Output -------------------------- // // ========================================================================= // @@ -13,7 +17,7 @@ ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next}, {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next}, {"^[-]+$", MR_Next}}); -static int AddContextCases() { +int AddContextCases() { AddCases(TC_ConsoleErr, { {"^%int-%int-%intT%int:%int:%int[-+]%int:%int$", MR_Default}, @@ -55,9 +59,12 @@ static int AddContextCases() { {{"Load Average: (%float, ){0,2}%float$", MR_Next}}); } AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}}); + AddCases(TC_JSONOut, {{"\"library_version\": \".*\",$", MR_Next}}); + AddCases(TC_JSONOut, {{"\"library_build_type\": \".*\",$", MR_Next}}); + AddCases(TC_JSONOut, {{"\"json_schema_version\": 1$", MR_Next}}); return 0; } -int dummy_register = AddContextCases(); +const int dummy_register = AddContextCases(); ADD_CASES(TC_CSVOut, {{"%csv_header"}}); // ========================================================================= // @@ -93,7 +100,9 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}}); void BM_bytes_per_second(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } state.SetBytesProcessed(1); } @@ -124,7 +133,9 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}}); void BM_items_per_second(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } state.SetItemsProcessed(1); } @@ -318,7 +329,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); // ========================================================================= // -// ------------------------ Testing Arg Name Output ----------------------- // +// ------------------------ Testing Arg Name Output ------------------------ // // ========================================================================= // void BM_arg_name(benchmark::State& state) { @@ -404,7 +415,9 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"}, void BM_Complexity_O1(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } state.SetComplexityN(state.range(0)); } @@ -1085,7 +1098,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_UserPercentStats/iterations:5/repeats:3/" {"^\"BM_UserPercentStats/iterations:5/repeats:3/" "manual_time_stddev\",%csv_report$"}, {"^\"BM_UserPercentStats/iterations:5/repeats:3/" - "manual_time_\",%csv_report$"}}); + "manual_time_\",%csv_cv_report$"}}); // ========================================================================= // // ------------------------- Testing StrEscape JSON ------------------------ // @@ -1119,9 +1132,13 @@ void BM_CSV_Format(benchmark::State& state) { } BENCHMARK(BM_CSV_Format); ADD_CASES(TC_CSVOut, {{"^\"BM_CSV_Format\",,,,,,,,true,\"\"\"freedom\"\"\"$"}}); +} // end namespace // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // // ========================================================================= // -int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } +int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + RunOutputTests(argc, argv); +} diff --git a/test/scoped_pause_test.cc b/test/scoped_pause_test.cc new file mode 100644 index 0000000000..93bcfc25a0 --- /dev/null +++ b/test/scoped_pause_test.cc @@ -0,0 +1,30 @@ + +#include +#include + +#include "benchmark/benchmark.h" +#include "output_test.h" + +// BM_ScopedPause sleeps for 10ms in a ScopedPauseTiming block. +// The reported time should be much less than 10ms. +void BM_ScopedPause(benchmark::State& state) { + for (auto _ : state) { + benchmark::ScopedPauseTiming pause(state); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + state.SetIterationTime(0.0); + } +} +BENCHMARK(BM_ScopedPause)->UseManualTime()->Iterations(1); + +void CheckResults(Results const& results) { + // Check that the real time is much less than the 10ms sleep time. + // Allow for up to 1ms of timing noise/overhead. + CHECK_FLOAT_RESULT_VALUE(results, "real_time", LT, 1e6, 0.0); +} +CHECK_BENCHMARK_RESULTS("BM_ScopedPause", &CheckResults); + +int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + RunOutputTests(argc, argv); + return 0; +} diff --git a/test/skip_with_error_test.cc b/test/skip_with_error_test.cc index 026d479133..d30c23e627 100644 --- a/test/skip_with_error_test.cc +++ b/test/skip_with_error_test.cc @@ -4,23 +4,27 @@ #include #include "../src/check.h" // NOTE: check.h is for internal use only! -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/reporter.h" +#include "benchmark/state.h" +#include "benchmark/utils.h" namespace { class TestReporter : public benchmark::ConsoleReporter { public: - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { + bool ReportContext(const Context& context) override { return ConsoleReporter::ReportContext(context); }; - virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + void ReportRuns(const std::vector& report) override { all_runs_.insert(all_runs_.end(), begin(report), end(report)); ConsoleReporter::ReportRuns(report); } TestReporter() {} - virtual ~TestReporter() {} + ~TestReporter() override {} mutable std::vector all_runs_; }; @@ -35,8 +39,9 @@ struct TestCase { void CheckRun(Run const& run) const { BM_CHECK(name == run.benchmark_name()) << "expected " << name << " got " << run.benchmark_name(); - BM_CHECK(error_occurred == run.error_occurred); - BM_CHECK(error_message == run.error_message); + BM_CHECK_EQ(error_occurred, + benchmark::internal::SkippedWithError == run.skipped); + BM_CHECK(error_message == run.skip_message); if (error_occurred) { // BM_CHECK(run.iterations == 0); } else { @@ -45,9 +50,11 @@ struct TestCase { } }; +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) std::vector ExpectedResults; -int AddCases(const char* base_name, std::initializer_list const& v) { +int AddCases(const std::string& base_name, + std::initializer_list const& v) { for (auto TC : v) { TC.name = base_name + TC.name; ExpectedResults.push_back(std::move(TC)); @@ -57,9 +64,7 @@ int AddCases(const char* base_name, std::initializer_list const& v) { #define CONCAT(x, y) CONCAT2(x, y) #define CONCAT2(x, y) x##y -#define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__) - -} // end namespace +#define ADD_CASES(...) const int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__) void BM_error_no_running(benchmark::State& state) { state.SkipWithError("error message"); @@ -95,11 +100,11 @@ BENCHMARK(BM_error_before_running_range_for); ADD_CASES("BM_error_before_running_range_for", {{"", true, "error message"}}); void BM_error_during_running(benchmark::State& state) { - int first_iter = true; + int first_iter = 1; while (state.KeepRunning()) { if (state.range(0) == 1 && state.thread_index() <= (state.threads() / 2)) { assert(first_iter); - first_iter = false; + first_iter = 0; state.SkipWithError("error message"); } else { state.PauseTiming(); @@ -141,10 +146,13 @@ ADD_CASES("BM_error_during_running_ranged_for", void BM_error_after_running(benchmark::State& state) { for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } - if (state.thread_index() <= (state.threads() / 2)) + if (state.thread_index() <= (state.threads() / 2)) { state.SkipWithError("error message"); + } } BENCHMARK(BM_error_after_running)->ThreadRange(1, 8); ADD_CASES("BM_error_after_running", {{"/threads:1", true, "error message"}, @@ -176,7 +184,18 @@ ADD_CASES("BM_error_while_paused", {{"/1/threads:1", true, "error message"}, {"/2/threads:4", false, ""}, {"/2/threads:8", false, ""}}); +void BM_malformed(benchmark::State& /*unused*/) { + // NOTE: empty body wanted. No thing else. +} +BENCHMARK(BM_malformed); +ADD_CASES("BM_malformed", + {{"", true, + "The benchmark didn't run, nor was it explicitly skipped. Please " + "call 'SkipWithXXX` in your benchmark as appropriate."}}); +} // end namespace + int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); benchmark::Initialize(&argc, argv); TestReporter test_reporter; diff --git a/test/spec_arg_test.cc b/test/spec_arg_test.cc index 68ab1351b3..393eca52f5 100644 --- a/test/spec_arg_test.cc +++ b/test/spec_arg_test.cc @@ -8,7 +8,10 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/reporter.h" +#include "benchmark/state.h" // Tests that we can override benchmark-spec value from FLAGS_benchmark_filter // with argument to RunSpecifiedBenchmarks(...). @@ -17,11 +20,11 @@ namespace { class TestReporter : public benchmark::ConsoleReporter { public: - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { + bool ReportContext(const Context& context) override { return ConsoleReporter::ReportContext(context); }; - virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + void ReportRuns(const std::vector& report) override { assert(report.size() == 1); matched_functions.push_back(report[0].run_name.function_name); ConsoleReporter::ReportRuns(report); @@ -29,7 +32,7 @@ class TestReporter : public benchmark::ConsoleReporter { TestReporter() {} - virtual ~TestReporter() {} + ~TestReporter() override {} const std::vector& GetMatchedFunctions() const { return matched_functions; @@ -39,22 +42,24 @@ class TestReporter : public benchmark::ConsoleReporter { std::vector matched_functions; }; -} // end namespace - -static void BM_NotChosen(benchmark::State& state) { +void BM_NotChosen(benchmark::State& state) { assert(false && "SHOULD NOT BE CALLED"); for (auto _ : state) { } } BENCHMARK(BM_NotChosen); -static void BM_Chosen(benchmark::State& state) { +void BM_Chosen(benchmark::State& state) { for (auto _ : state) { } } BENCHMARK(BM_Chosen); +} // end namespace + int main(int argc, char** argv) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + const std::string flag = "BM_NotChosen"; // Verify that argv specify --benchmark_filter=BM_NotChosen. diff --git a/test/spec_arg_verbosity_test.cc b/test/spec_arg_verbosity_test.cc index 8f8eb6d37c..49aadd9f37 100644 --- a/test/spec_arg_verbosity_test.cc +++ b/test/spec_arg_verbosity_test.cc @@ -2,16 +2,22 @@ #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +namespace { // Tests that the user specified verbosity level can be get. -static void BM_Verbosity(benchmark::State& state) { +void BM_Verbosity(benchmark::State& state) { for (auto _ : state) { } } BENCHMARK(BM_Verbosity); +} // end namespace int main(int argc, char** argv) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + const int32_t flagv = 42; // Verify that argv specify --v=42. diff --git a/test/state_assembly_test.cc b/test/state_assembly_test.cc index 7ddbb3b2a9..5efceedd9e 100644 --- a/test/state_assembly_test.cc +++ b/test/state_assembly_test.cc @@ -1,7 +1,9 @@ -#include +#include "benchmark/state.h" +#include "benchmark/utils.h" #ifdef __clang__ #pragma clang diagnostic ignored "-Wreturn-type" +#pragma clang diagnostic ignored "-Wmissing-prototypes" #endif // clang-format off diff --git a/test/statistics_gtest.cc b/test/statistics_gtest.cc index 1de2d87d4b..48c77260fd 100644 --- a/test/statistics_gtest.cc +++ b/test/statistics_gtest.cc @@ -28,8 +28,8 @@ TEST(StatisticsTest, StdDev) { TEST(StatisticsTest, CV) { EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({101, 101, 101, 101}), 0.0); EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({1, 2, 3}), 1. / 2.); - EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({2.5, 2.4, 3.3, 4.2, 5.1}), - 0.32888184094918121); + ASSERT_NEAR(benchmark::StatisticsCV({2.5, 2.4, 3.3, 4.2, 5.1}), + 0.32888184094918121, 1e-15); } } // end namespace diff --git a/test/string_util_gtest.cc b/test/string_util_gtest.cc index 698f2d43eb..5a9a09e19b 100644 --- a/test/string_util_gtest.cc +++ b/test/string_util_gtest.cc @@ -1,27 +1,30 @@ //===---------------------------------------------------------------------===// -// statistics_test - Unit tests for src/statistics.cc +// string_util_test - Unit tests for src/string_util.cc //===---------------------------------------------------------------------===// +#include + #include "../src/internal_macros.h" #include "../src/string_util.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" namespace { TEST(StringUtilTest, stoul) { { size_t pos = 0; - EXPECT_EQ(0ul, benchmark::stoul("0", &pos)); - EXPECT_EQ(1ul, pos); + EXPECT_EQ(0UL, benchmark::stoul("0", &pos)); + EXPECT_EQ(1UL, pos); } { size_t pos = 0; - EXPECT_EQ(7ul, benchmark::stoul("7", &pos)); - EXPECT_EQ(1ul, pos); + EXPECT_EQ(7UL, benchmark::stoul("7", &pos)); + EXPECT_EQ(1UL, pos); } { size_t pos = 0; - EXPECT_EQ(135ul, benchmark::stoul("135", &pos)); - EXPECT_EQ(3ul, pos); + EXPECT_EQ(135UL, benchmark::stoul("135", &pos)); + EXPECT_EQ(3UL, pos); } #if ULONG_MAX == 0xFFFFFFFFul { @@ -32,112 +35,125 @@ TEST(StringUtilTest, stoul) { #elif ULONG_MAX == 0xFFFFFFFFFFFFFFFFul { size_t pos = 0; - EXPECT_EQ(0xFFFFFFFFFFFFFFFFul, + EXPECT_EQ(0xFFFFFFFFFFFFFFFFUL, benchmark::stoul("18446744073709551615", &pos)); - EXPECT_EQ(20ul, pos); + EXPECT_EQ(20UL, pos); } #endif { size_t pos = 0; - EXPECT_EQ(10ul, benchmark::stoul("1010", &pos, 2)); - EXPECT_EQ(4ul, pos); + EXPECT_EQ(10UL, benchmark::stoul("1010", &pos, 2)); + EXPECT_EQ(4UL, pos); } { size_t pos = 0; - EXPECT_EQ(520ul, benchmark::stoul("1010", &pos, 8)); - EXPECT_EQ(4ul, pos); + EXPECT_EQ(520UL, benchmark::stoul("1010", &pos, 8)); + EXPECT_EQ(4UL, pos); } { size_t pos = 0; - EXPECT_EQ(1010ul, benchmark::stoul("1010", &pos, 10)); - EXPECT_EQ(4ul, pos); + EXPECT_EQ(1010UL, benchmark::stoul("1010", &pos, 10)); + EXPECT_EQ(4UL, pos); } { size_t pos = 0; - EXPECT_EQ(4112ul, benchmark::stoul("1010", &pos, 16)); - EXPECT_EQ(4ul, pos); + EXPECT_EQ(4112UL, benchmark::stoul("1010", &pos, 16)); + EXPECT_EQ(4UL, pos); } { size_t pos = 0; - EXPECT_EQ(0xBEEFul, benchmark::stoul("BEEF", &pos, 16)); - EXPECT_EQ(4ul, pos); + EXPECT_EQ(0xBEEFUL, benchmark::stoul("BEEF", &pos, 16)); + EXPECT_EQ(4UL, pos); } #ifndef BENCHMARK_HAS_NO_EXCEPTIONS - { ASSERT_THROW(benchmark::stoul("this is a test"), std::invalid_argument); } + { + ASSERT_THROW(std::ignore = benchmark::stoul("this is a test"), + std::invalid_argument); + } #endif } -TEST(StringUtilTest, stoi){{size_t pos = 0; -EXPECT_EQ(0, benchmark::stoi("0", &pos)); -EXPECT_EQ(1ul, pos); -} // namespace -{ - size_t pos = 0; - EXPECT_EQ(-17, benchmark::stoi("-17", &pos)); - EXPECT_EQ(3ul, pos); -} -{ - size_t pos = 0; - EXPECT_EQ(1357, benchmark::stoi("1357", &pos)); - EXPECT_EQ(4ul, pos); -} -{ - size_t pos = 0; - EXPECT_EQ(10, benchmark::stoi("1010", &pos, 2)); - EXPECT_EQ(4ul, pos); -} -{ - size_t pos = 0; - EXPECT_EQ(520, benchmark::stoi("1010", &pos, 8)); - EXPECT_EQ(4ul, pos); -} -{ - size_t pos = 0; - EXPECT_EQ(1010, benchmark::stoi("1010", &pos, 10)); - EXPECT_EQ(4ul, pos); -} -{ - size_t pos = 0; - EXPECT_EQ(4112, benchmark::stoi("1010", &pos, 16)); - EXPECT_EQ(4ul, pos); -} -{ - size_t pos = 0; - EXPECT_EQ(0xBEEF, benchmark::stoi("BEEF", &pos, 16)); - EXPECT_EQ(4ul, pos); -} +TEST(StringUtilTest, stoi) { + { + size_t pos = 0; + EXPECT_EQ(0, benchmark::stoi("0", &pos)); + EXPECT_EQ(1UL, pos); + } // namespace + { + size_t pos = 0; + EXPECT_EQ(-17, benchmark::stoi("-17", &pos)); + EXPECT_EQ(3UL, pos); + } + { + size_t pos = 0; + EXPECT_EQ(1357, benchmark::stoi("1357", &pos)); + EXPECT_EQ(4UL, pos); + } + { + size_t pos = 0; + EXPECT_EQ(10, benchmark::stoi("1010", &pos, 2)); + EXPECT_EQ(4UL, pos); + } + { + size_t pos = 0; + EXPECT_EQ(520, benchmark::stoi("1010", &pos, 8)); + EXPECT_EQ(4UL, pos); + } + { + size_t pos = 0; + EXPECT_EQ(1010, benchmark::stoi("1010", &pos, 10)); + EXPECT_EQ(4UL, pos); + } + { + size_t pos = 0; + EXPECT_EQ(4112, benchmark::stoi("1010", &pos, 16)); + EXPECT_EQ(4UL, pos); + } + { + size_t pos = 0; + EXPECT_EQ(0xBEEF, benchmark::stoi("BEEF", &pos, 16)); + EXPECT_EQ(4UL, pos); + } #ifndef BENCHMARK_HAS_NO_EXCEPTIONS -{ ASSERT_THROW(benchmark::stoi("this is a test"), std::invalid_argument); } + { + ASSERT_THROW(std::ignore = benchmark::stoi("this is a test"), + std::invalid_argument); + } #endif } -TEST(StringUtilTest, stod){{size_t pos = 0; -EXPECT_EQ(0.0, benchmark::stod("0", &pos)); -EXPECT_EQ(1ul, pos); -} -{ - size_t pos = 0; - EXPECT_EQ(-84.0, benchmark::stod("-84", &pos)); - EXPECT_EQ(3ul, pos); -} -{ - size_t pos = 0; - EXPECT_EQ(1234.0, benchmark::stod("1234", &pos)); - EXPECT_EQ(4ul, pos); -} -{ - size_t pos = 0; - EXPECT_EQ(1.5, benchmark::stod("1.5", &pos)); - EXPECT_EQ(3ul, pos); -} -{ - size_t pos = 0; - /* Note: exactly representable as double */ - EXPECT_EQ(-1.25e+9, benchmark::stod("-1.25e+9", &pos)); - EXPECT_EQ(8ul, pos); -} +TEST(StringUtilTest, stod) { + { + size_t pos = 0; + EXPECT_EQ(0.0, benchmark::stod("0", &pos)); + EXPECT_EQ(1UL, pos); + } + { + size_t pos = 0; + EXPECT_EQ(-84.0, benchmark::stod("-84", &pos)); + EXPECT_EQ(3UL, pos); + } + { + size_t pos = 0; + EXPECT_EQ(1234.0, benchmark::stod("1234", &pos)); + EXPECT_EQ(4UL, pos); + } + { + size_t pos = 0; + EXPECT_EQ(1.5, benchmark::stod("1.5", &pos)); + EXPECT_EQ(3UL, pos); + } + { + size_t pos = 0; + /* Note: exactly representable as double */ + EXPECT_EQ(-1.25e+9, benchmark::stod("-1.25e+9", &pos)); + EXPECT_EQ(8UL, pos); + } #ifndef BENCHMARK_HAS_NO_EXCEPTIONS -{ ASSERT_THROW(benchmark::stod("this is a test"), std::invalid_argument); } + { + ASSERT_THROW(std::ignore = benchmark::stod("this is a test"), + std::invalid_argument); + } #endif } @@ -149,4 +165,39 @@ TEST(StringUtilTest, StrSplit) { std::vector({"hello", "there", "is", "more"})); } +using HumanReadableFixture = ::testing::TestWithParam< + std::tuple>; + +INSTANTIATE_TEST_SUITE_P( + HumanReadableTests, HumanReadableFixture, + ::testing::Values( + std::make_tuple(0.0, benchmark::Counter::kIs1024, "0"), + std::make_tuple(999.0, benchmark::Counter::kIs1024, "999"), + std::make_tuple(1000.0, benchmark::Counter::kIs1024, "1000"), + std::make_tuple(1024.0, benchmark::Counter::kIs1024, "1Ki"), + std::make_tuple(1000 * 1000.0, benchmark::Counter::kIs1024, + "976\\.56.Ki"), + std::make_tuple(1024 * 1024.0, benchmark::Counter::kIs1024, "1Mi"), + std::make_tuple(1000 * 1000 * 1000.0, benchmark::Counter::kIs1024, + "953\\.674Mi"), + std::make_tuple(1024 * 1024 * 1024.0, benchmark::Counter::kIs1024, + "1Gi"), + std::make_tuple(0.0, benchmark::Counter::kIs1000, "0"), + std::make_tuple(999.0, benchmark::Counter::kIs1000, "999"), + std::make_tuple(1000.0, benchmark::Counter::kIs1000, "1k"), + std::make_tuple(1024.0, benchmark::Counter::kIs1000, "1.024k"), + std::make_tuple(1000 * 1000.0, benchmark::Counter::kIs1000, "1M"), + std::make_tuple(1024 * 1024.0, benchmark::Counter::kIs1000, + "1\\.04858M"), + std::make_tuple(1000 * 1000 * 1000.0, benchmark::Counter::kIs1000, + "1G"), + std::make_tuple(1024 * 1024 * 1024.0, benchmark::Counter::kIs1000, + "1\\.07374G"))); + +TEST_P(HumanReadableFixture, HumanReadableNumber) { + std::string str = benchmark::HumanReadableNumber(std::get<0>(GetParam()), + std::get<1>(GetParam())); + ASSERT_THAT(str, ::testing::MatchesRegex(std::get<2>(GetParam()))); +} + } // end namespace diff --git a/test/templated_fixture_method_test.cc b/test/templated_fixture_method_test.cc new file mode 100644 index 0000000000..3de726baa1 --- /dev/null +++ b/test/templated_fixture_method_test.cc @@ -0,0 +1,28 @@ + +#include +#include + +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" + +template +class MyFixture : public ::benchmark::Fixture { + public: + MyFixture() : data(0) {} + + T data; + + using type = T; +}; + +BENCHMARK_TEMPLATE_METHOD_F(MyFixture, Foo)(benchmark::State& st) { + for (auto _ : st) { + this->data += typename Base::type(1); + } +} + +BENCHMARK_TEMPLATE_INSTANTIATE_F(MyFixture, Foo, int); +BENCHMARK_TEMPLATE_INSTANTIATE_F(MyFixture, Foo, double); + +BENCHMARK_MAIN(); diff --git a/test/templated_fixture_test.cc b/test/templated_fixture_test.cc index af239c3a72..44108dda9c 100644 --- a/test/templated_fixture_test.cc +++ b/test/templated_fixture_test.cc @@ -2,7 +2,9 @@ #include #include -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" template class MyFixture : public ::benchmark::Fixture { diff --git a/test/time_unit_gtest.cc b/test/time_unit_gtest.cc index ae53743285..1d4d2d086c 100644 --- a/test/time_unit_gtest.cc +++ b/test/time_unit_gtest.cc @@ -1,4 +1,5 @@ -#include "../include/benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/state.h" #include "gtest/gtest.h" namespace benchmark { @@ -6,10 +7,10 @@ namespace internal { namespace { -class DummyBenchmark : public Benchmark { +class DummyBenchmark : public benchmark::Benchmark { public: DummyBenchmark() : Benchmark("dummy") {} - virtual void Run(State&) override {} + void Run(State& /*state*/) override {} }; TEST(DefaultTimeUnitTest, TimeUnitIsNotSet) { diff --git a/test/user_counters_tabular_test.cc b/test/user_counters_tabular_test.cc index 45ac043d51..f173f1ba1d 100644 --- a/test/user_counters_tabular_test.cc +++ b/test/user_counters_tabular_test.cc @@ -1,9 +1,14 @@ #undef NDEBUG -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/counter.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/utils.h" #include "output_test.h" +namespace { // @todo: this checks the full output at once; the rule for // CounterSet1 was failing because it was not matching "^[-]+$". // @todo: check that the counters are vertically aligned. @@ -63,6 +68,10 @@ ADD_CASES(TC_CSVOut, {{"%csv_header," void BM_Counters_Tabular(benchmark::State& state) { for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters.insert({ @@ -330,7 +339,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/repeats:2/threads:1_stddev\",%csv_report," "%float,%float,%float,%float,%float,%float$"}}); ADD_CASES(TC_CSVOut, - {{"^\"BM_Counters_Tabular/repeats:2/threads:1_cv\",%csv_report," + {{"^\"BM_Counters_Tabular/repeats:2/threads:1_cv\",%csv_cv_report," "%float,%float,%float,%float,%float,%float$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report," @@ -348,7 +357,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/repeats:2/threads:2_stddev\",%csv_report," "%float,%float,%float,%float,%float,%float$"}}); ADD_CASES(TC_CSVOut, - {{"^\"BM_Counters_Tabular/repeats:2/threads:2_cv\",%csv_report," + {{"^\"BM_Counters_Tabular/repeats:2/threads:2_cv\",%csv_cv_report," "%float,%float,%float,%float,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument // to CHECK_BENCHMARK_RESULTS() @@ -372,7 +381,9 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:2$", void BM_CounterRates_Tabular(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters.insert({ @@ -411,7 +422,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report," // VS2013 does not allow this function to be passed as a lambda argument // to CHECK_BENCHMARK_RESULTS() void CheckTabularRate(Results const& e) { - double t = e.DurationCPUTime(); + double t = e.DurationCPUTime() / e.NumThreads(); CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1. / t, 0.001); CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2. / t, 0.001); CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4. / t, 0.001); @@ -549,9 +560,13 @@ void CheckSet2(Results const& e) { CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40); } CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2); +} // end namespace // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // // ========================================================================= // -int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } +int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + RunOutputTests(argc, argv); +} diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index 1cc74552a1..9253d594f3 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -1,7 +1,11 @@ #undef NDEBUG -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/counter.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/utils.h" #include "output_test.h" // ========================================================================= // @@ -21,7 +25,7 @@ ADD_CASES(TC_CSVOut, {{"%csv_header,\"bar\",\"foo\""}}); // ========================================================================= // // ------------------------- Simple Counters Output ------------------------ // // ========================================================================= // - +namespace { void BM_Counters_Simple(benchmark::State& state) { for (auto _ : state) { } @@ -56,6 +60,7 @@ void CheckSimple(Results const& e) { CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. * its, 0.001); } CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", &CheckSimple); +} // end namespace // ========================================================================= // // --------------------- Counters+Items+Bytes/s Output --------------------- // @@ -63,11 +68,12 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", &CheckSimple); namespace { int num_calls1 = 0; -} void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } state.counters["foo"] = 1; state.counters["bar"] = ++num_calls1; @@ -110,15 +116,18 @@ void CheckBytesAndItemsPSec(Results const& e) { } CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec", &CheckBytesAndItemsPSec); +} // end namespace // ========================================================================= // // ------------------------- Rate Counters Output -------------------------- // // ========================================================================= // - +namespace { void BM_Counters_Rate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate}; @@ -153,15 +162,19 @@ void CheckRate(Results const& e) { CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / t, 0.001); } CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", &CheckRate); +} // end namespace // ========================================================================= // // ----------------------- Inverted Counters Output ------------------------ // // ========================================================================= // +namespace { void BM_Invert(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{0.0001, bm::Counter::kInvert}; @@ -193,16 +206,19 @@ void CheckInvert(Results const& e) { CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 0.0001, 0.0001); } CHECK_BENCHMARK_RESULTS("BM_Invert", &CheckInvert); +} // end namespace // ========================================================================= // -// ------------------------- InvertedRate Counters Output -// -------------------------- // +// --------------------- InvertedRate Counters Output ---------------------- // // ========================================================================= // +namespace { void BM_Counters_InvertedRate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = @@ -240,11 +256,13 @@ void CheckInvertedRate(Results const& e) { CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, t / 8192.0, 0.001); } CHECK_BENCHMARK_RESULTS("BM_Counters_InvertedRate", &CheckInvertedRate); +} // end namespace // ========================================================================= // // ------------------------- Thread Counters Output ------------------------ // // ========================================================================= // +namespace { void BM_Counters_Threads(benchmark::State& state) { for (auto _ : state) { } @@ -280,11 +298,13 @@ void CheckThreads(Results const& e) { CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2 * e.NumThreads()); } CHECK_BENCHMARK_RESULTS("BM_Counters_Threads/threads:%int", &CheckThreads); +} // end namespace // ========================================================================= // // ---------------------- ThreadAvg Counters Output ------------------------ // // ========================================================================= // +namespace { void BM_Counters_AvgThreads(benchmark::State& state) { for (auto _ : state) { } @@ -322,15 +342,19 @@ void CheckAvgThreads(Results const& e) { } CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreads/threads:%int", &CheckAvgThreads); +} // end namespace // ========================================================================= // // ---------------------- ThreadAvg Counters Output ------------------------ // // ========================================================================= // +namespace { void BM_Counters_AvgThreadsRate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreadsRate}; @@ -361,16 +385,20 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_AvgThreadsRate/" // VS2013 does not allow this function to be passed as a lambda argument // to CHECK_BENCHMARK_RESULTS() void CheckAvgThreadsRate(Results const& e) { - CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / e.DurationCPUTime(), 0.001); - CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / e.DurationCPUTime(), 0.001); + // this (and not real time) is the time used + double t = e.DurationCPUTime() / e.NumThreads(); + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / t, 0.001); } CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreadsRate/threads:%int", &CheckAvgThreadsRate); +} // end namespace // ========================================================================= // // ------------------- IterationInvariant Counters Output ------------------ // // ========================================================================= // +namespace { void BM_Counters_IterationInvariant(benchmark::State& state) { for (auto _ : state) { } @@ -409,15 +437,19 @@ void CheckIterationInvariant(Results const& e) { } CHECK_BENCHMARK_RESULTS("BM_Counters_IterationInvariant", &CheckIterationInvariant); +} // end namespace // ========================================================================= // // ----------------- IterationInvariantRate Counters Output ---------------- // // ========================================================================= // +namespace { void BM_Counters_kIsIterationInvariantRate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = @@ -458,11 +490,13 @@ void CheckIsIterationInvariantRate(Results const& e) { } CHECK_BENCHMARK_RESULTS("BM_Counters_kIsIterationInvariantRate", &CheckIsIterationInvariantRate); +} // end namespace // ========================================================================= // -// ------------------- AvgIterations Counters Output ------------------ // +// --------------------- AvgIterations Counters Output --------------------- // // ========================================================================= // +namespace { void BM_Counters_AvgIterations(benchmark::State& state) { for (auto _ : state) { } @@ -500,15 +534,19 @@ void CheckAvgIterations(Results const& e) { CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / its, 0.001); } CHECK_BENCHMARK_RESULTS("BM_Counters_AvgIterations", &CheckAvgIterations); +} // end namespace // ========================================================================= // -// ----------------- AvgIterationsRate Counters Output ---------------- // +// ------------------- AvgIterationsRate Counters Output ------------------- // // ========================================================================= // +namespace { void BM_Counters_kAvgIterationsRate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterationsRate}; @@ -547,9 +585,13 @@ void CheckAvgIterationsRate(Results const& e) { } CHECK_BENCHMARK_RESULTS("BM_Counters_kAvgIterationsRate", &CheckAvgIterationsRate); +} // end namespace // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // // ========================================================================= // -int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } +int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + RunOutputTests(argc, argv); +} diff --git a/test/user_counters_thousands_test.cc b/test/user_counters_thousands_test.cc index a42683b32f..58170e98b9 100644 --- a/test/user_counters_thousands_test.cc +++ b/test/user_counters_thousands_test.cc @@ -1,9 +1,13 @@ #undef NDEBUG -#include "benchmark/benchmark.h" +#include "benchmark/benchmark_api.h" +#include "benchmark/counter.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" #include "output_test.h" +namespace { // ========================================================================= // // ------------------------ Thousands Customisation ------------------------ // // ========================================================================= // @@ -16,13 +20,13 @@ void BM_Counters_Thousands(benchmark::State& state) { {"t0_1000000DefaultBase", bm::Counter(1000 * 1000, bm::Counter::kDefaults)}, {"t1_1000000Base1000", bm::Counter(1000 * 1000, bm::Counter::kDefaults, - benchmark::Counter::OneK::kIs1000)}, + bm::Counter::OneK::kIs1000)}, {"t2_1000000Base1024", bm::Counter(1000 * 1000, bm::Counter::kDefaults, - benchmark::Counter::OneK::kIs1024)}, + bm::Counter::OneK::kIs1024)}, {"t3_1048576Base1000", bm::Counter(1024 * 1024, bm::Counter::kDefaults, - benchmark::Counter::OneK::kIs1000)}, + bm::Counter::OneK::kIs1000)}, {"t4_1048576Base1024", bm::Counter(1024 * 1024, bm::Counter::kDefaults, - benchmark::Counter::OneK::kIs1024)}, + bm::Counter::OneK::kIs1024)}, }); } BENCHMARK(BM_Counters_Thousands)->Repetitions(2); @@ -30,21 +34,21 @@ ADD_CASES( TC_ConsoleOut, { {"^BM_Counters_Thousands/repeats:2 %console_report " - "t0_1000000DefaultBase=1000k " - "t1_1000000Base1000=1000k t2_1000000Base1024=976.56[23]k " - "t3_1048576Base1000=1048.58k t4_1048576Base1024=1024k$"}, + "t0_1000000DefaultBase=1M " + "t1_1000000Base1000=1M t2_1000000Base1024=976.56[23]Ki " + "t3_1048576Base1000=1.04858M t4_1048576Base1024=1Mi$"}, {"^BM_Counters_Thousands/repeats:2 %console_report " - "t0_1000000DefaultBase=1000k " - "t1_1000000Base1000=1000k t2_1000000Base1024=976.56[23]k " - "t3_1048576Base1000=1048.58k t4_1048576Base1024=1024k$"}, + "t0_1000000DefaultBase=1M " + "t1_1000000Base1000=1M t2_1000000Base1024=976.56[23]Ki " + "t3_1048576Base1000=1.04858M t4_1048576Base1024=1Mi$"}, {"^BM_Counters_Thousands/repeats:2_mean %console_report " - "t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k " - "t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k " - "t4_1048576Base1024=1024k$"}, + "t0_1000000DefaultBase=1M t1_1000000Base1000=1M " + "t2_1000000Base1024=976.56[23]Ki t3_1048576Base1000=1.04858M " + "t4_1048576Base1024=1Mi$"}, {"^BM_Counters_Thousands/repeats:2_median %console_report " - "t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k " - "t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k " - "t4_1048576Base1024=1024k$"}, + "t0_1000000DefaultBase=1M t1_1000000Base1000=1M " + "t2_1000000Base1024=976.56[23]Ki t3_1048576Base1000=1.04858M " + "t4_1048576Base1024=1Mi$"}, {"^BM_Counters_Thousands/repeats:2_stddev %console_time_only_report [ " "]*2 t0_1000000DefaultBase=0 t1_1000000Base1000=0 " "t2_1000000Base1024=0 t3_1048576Base1000=0 t4_1048576Base1024=0$"}, @@ -166,8 +170,9 @@ ADD_CASES( // VS2013 does not allow this function to be passed as a lambda argument // to CHECK_BENCHMARK_RESULTS() void CheckThousands(Results const& e) { - if (e.name != "BM_Counters_Thousands/repeats:2") + if (e.name != "BM_Counters_Thousands/repeats:2") { return; // Do not check the aggregates! + } // check that the values are within 0.01% of the expected values CHECK_FLOAT_COUNTER_VALUE(e, "t0_1000000DefaultBase", EQ, 1000 * 1000, @@ -178,9 +183,13 @@ void CheckThousands(Results const& e) { CHECK_FLOAT_COUNTER_VALUE(e, "t4_1048576Base1024", EQ, 1024 * 1024, 0.0001); } CHECK_BENCHMARK_RESULTS("BM_Counters_Thousands", &CheckThousands); +} // end namespace // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // // ========================================================================= // -int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } +int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + RunOutputTests(argc, argv); +} diff --git a/test/user_counters_threads_test.cc b/test/user_counters_threads_test.cc new file mode 100644 index 0000000000..9fd01a646c --- /dev/null +++ b/test/user_counters_threads_test.cc @@ -0,0 +1,625 @@ + +#undef NDEBUG + +#include "benchmark/benchmark_api.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/utils.h" +#include "output_test.h" + +// ========================================================================= // +// ---------------------- Testing Prologue Output -------------------------- // +// ========================================================================= // + +// clang-format off + +ADD_CASES(TC_ConsoleOut, + {{"^[-]+$", MR_Next}, + {"^Benchmark %s Time %s CPU %s Iterations UserCounters...$", MR_Next}, + {"^[-]+$", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"%csv_header,\"bar\",\"foo\""}}); + +// clang-format on + +// ========================================================================= // +// ------------------------- Simple Counters Output ------------------------ // +// ========================================================================= // + +namespace { +void BM_Counters_Simple(benchmark::State& state) { + for (auto _ : state) { + } + state.counters["foo"] = 1; + state.counters["bar"] = 2 * static_cast(state.iterations()); +} +BENCHMARK(BM_Counters_Simple)->ThreadRange(1, 8); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple/threads:%int %console_report " + "bar=%hrfloat foo=%hrfloat$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Simple/threads:%int\",$"}, + {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Counters_Simple/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": %int,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES( + TC_CSVOut, + {{"^\"BM_Counters_Simple/threads:%int\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckSimple(Results const& e) { + double its = e.NumIterations(); + CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1 * e.NumThreads()); + // check that the value of bar is within 0.1% of the expected value + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. * its, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_Simple/threads:%int", &CheckSimple); +} // end namespace + +// ========================================================================= // +// --------------------- Counters+Items+Bytes/s Output --------------------- // +// ========================================================================= // + +namespace { +void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) { + for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); + } + state.counters["foo"] = 1; + state.SetBytesProcessed(364); + state.SetItemsProcessed(150); +} +BENCHMARK(BM_Counters_WithBytesAndItemsPSec)->ThreadRange(1, 8); +ADD_CASES(TC_ConsoleOut, + {{"^BM_Counters_WithBytesAndItemsPSec/threads:%int %console_report " + "bytes_per_second=%hrfloat/s " + "foo=%hrfloat items_per_second=%hrfloat/s$"}}); +ADD_CASES( + TC_JSONOut, + {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec/threads:%int\",$"}, + {"\"family_index\": 1,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Counters_WithBytesAndItemsPSec/threads:%int\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": %int,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bytes_per_second\": %float,$", MR_Next}, + {"\"foo\": %float,$", MR_Next}, + {"\"items_per_second\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec/threads:%int\"," + "%csv_bytes_items_report,,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckBytesAndItemsPSec(Results const& e) { + // this (and not real time) is the time used + double t = e.DurationCPUTime() / e.NumThreads(); + CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1 * e.NumThreads()); + // check that the values are within 0.1% of the expected values + CHECK_FLOAT_RESULT_VALUE(e, "bytes_per_second", EQ, + (364. * e.NumThreads()) / t, 0.001); + CHECK_FLOAT_RESULT_VALUE(e, "items_per_second", EQ, + (150. * e.NumThreads()) / t, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec/threads:%int", + &CheckBytesAndItemsPSec); +} // end namespace + +// ========================================================================= // +// ------------------------- Rate Counters Output -------------------------- // +// ========================================================================= // +namespace { +void BM_Counters_Rate(benchmark::State& state) { + for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); + } + namespace bm = benchmark; + state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate}; + state.counters["bar"] = bm::Counter{2, bm::Counter::kIsRate}; +} +BENCHMARK(BM_Counters_Rate)->ThreadRange(1, 8); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Rate/threads:%int %console_report " + "bar=%hrfloat/s foo=%hrfloat/s$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Rate/threads:%int\",$"}, + {"\"family_index\": 2,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Counters_Rate/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": %int,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Counters_Rate/threads:%int\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckRate(Results const& e) { + // this (and not real time) is the time used + double t = e.DurationCPUTime() / e.NumThreads(); + // check that the values are within 0.1% of the expected values + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, (1. * e.NumThreads()) / t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, (2. * e.NumThreads()) / t, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_Rate/threads:%int", &CheckRate); +} // end namespace + +// ========================================================================= // +// ----------------------- Inverted Counters Output ------------------------ // +// ========================================================================= // + +namespace { +void BM_Invert(benchmark::State& state) { + for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); + } + namespace bm = benchmark; + state.counters["foo"] = bm::Counter{0.0001, bm::Counter::kInvert}; + state.counters["bar"] = bm::Counter{10000, bm::Counter::kInvert}; +} +BENCHMARK(BM_Invert)->ThreadRange(1, 8); +ADD_CASES( + TC_ConsoleOut, + {{"^BM_Invert/threads:%int %console_report bar=%hrfloatu foo=%hrfloatk$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Invert/threads:%int\",$"}, + {"\"family_index\": 3,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Invert/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": %int,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Invert/threads:%int\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckInvert(Results const& e) { + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / (0.0001 * e.NumThreads()), + 0.0001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 1. / (10000 * e.NumThreads()), + 0.0001); +} +CHECK_BENCHMARK_RESULTS("BM_Invert/threads:%int", &CheckInvert); +} // end namespace + +// ========================================================================= // +// --------------------- InvertedRate Counters Output ---------------------- // +// ========================================================================= // + +namespace { +void BM_Counters_InvertedRate(benchmark::State& state) { + for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); + } + namespace bm = benchmark; + state.counters["foo"] = + bm::Counter{1, bm::Counter::kIsRate | bm::Counter::kInvert}; + state.counters["bar"] = + bm::Counter{8192, bm::Counter::kIsRate | bm::Counter::kInvert}; +} +BENCHMARK(BM_Counters_InvertedRate)->ThreadRange(1, 8); +ADD_CASES(TC_ConsoleOut, + {{"^BM_Counters_InvertedRate/threads:%int %console_report " + "bar=%hrfloats foo=%hrfloats$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_InvertedRate/threads:%int\",$"}, + {"\"family_index\": 4,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Counters_InvertedRate/threads:%int\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": %int,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_InvertedRate/" + "threads:%int\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckInvertedRate(Results const& e) { + // this (and not real time) is the time used + double t = e.DurationCPUTime() / e.NumThreads(); + // check that the values are within 0.1% of the expected values + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, t / (e.NumThreads()), 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, t / (8192.0 * e.NumThreads()), 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_InvertedRate/threads:%int", + &CheckInvertedRate); +} // end namespace + +// ========================================================================= // +// ------------------------- Thread Counters Output ------------------------ // +// ========================================================================= // + +namespace { +void BM_Counters_Threads(benchmark::State& state) { + for (auto _ : state) { + } + state.counters["foo"] = 1; + state.counters["bar"] = 2; +} +BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report " + "bar=%hrfloat foo=%hrfloat$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, + {"\"family_index\": 5,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Counters_Threads/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": %int,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES( + TC_CSVOut, + {{"^\"BM_Counters_Threads/threads:%int\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckThreads(Results const& e) { + CHECK_COUNTER_VALUE(e, int, "foo", EQ, e.NumThreads()); + CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2 * e.NumThreads()); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_Threads/threads:%int", &CheckThreads); +} // end namespace + +// ========================================================================= // +// ---------------------- ThreadAvg Counters Output ------------------------ // +// ========================================================================= // + +namespace { +void BM_Counters_AvgThreads(benchmark::State& state) { + for (auto _ : state) { + } + namespace bm = benchmark; + state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreads}; + state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreads}; +} +BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int " + "%console_report bar=%hrfloat foo=%hrfloat$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, + {"\"family_index\": 6,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Counters_AvgThreads/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": %int,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES( + TC_CSVOut, + {{"^\"BM_Counters_AvgThreads/threads:%int\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckAvgThreads(Results const& e) { + CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); + CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreads/threads:%int", + &CheckAvgThreads); +} // end namespace + +// ========================================================================= // +// ---------------------- ThreadAvg Counters Output ------------------------ // +// ========================================================================= // + +namespace { +void BM_Counters_AvgThreadsRate(benchmark::State& state) { + for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); + } + namespace bm = benchmark; + state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreadsRate}; + state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreadsRate}; +} +BENCHMARK(BM_Counters_AvgThreadsRate)->ThreadRange(1, 8); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int " + "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"}, + {"\"family_index\": 7,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": %int,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_AvgThreadsRate/" + "threads:%int\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckAvgThreadsRate(Results const& e) { + // this (and not real time) is the time used + double t = e.DurationCPUTime() / e.NumThreads(); + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / t, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreadsRate/threads:%int", + &CheckAvgThreadsRate); +} // end namespace + +// ========================================================================= // +// ------------------- IterationInvariant Counters Output ------------------ // +// ========================================================================= // + +namespace { +void BM_Counters_IterationInvariant(benchmark::State& state) { + for (auto _ : state) { + } + namespace bm = benchmark; + state.counters["foo"] = bm::Counter{1, bm::Counter::kIsIterationInvariant}; + state.counters["bar"] = bm::Counter{2, bm::Counter::kIsIterationInvariant}; +} +BENCHMARK(BM_Counters_IterationInvariant)->ThreadRange(1, 8); +ADD_CASES(TC_ConsoleOut, + {{"^BM_Counters_IterationInvariant/threads:%int %console_report " + "bar=%hrfloat foo=%hrfloat$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_IterationInvariant/threads:%int\",$"}, + {"\"family_index\": 8,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Counters_IterationInvariant/threads:%int\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": %int,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_IterationInvariant/" + "threads:%int\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckIterationInvariant(Results const& e) { + double its = e.NumIterations(); + // check that the values are within 0.1% of the expected value + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, its * e.NumThreads(), 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. * its * e.NumThreads(), 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_IterationInvariant/threads:%int", + &CheckIterationInvariant); +} // end namespace + +// ========================================================================= // +// ----------------- IterationInvariantRate Counters Output ---------------- // +// ========================================================================= // + +namespace { +void BM_Counters_kIsIterationInvariantRate(benchmark::State& state) { + for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); + } + namespace bm = benchmark; + state.counters["foo"] = + bm::Counter{1, bm::Counter::kIsIterationInvariantRate}; + state.counters["bar"] = + bm::Counter{2, bm::Counter::kIsRate | bm::Counter::kIsIterationInvariant}; +} +BENCHMARK(BM_Counters_kIsIterationInvariantRate)->ThreadRange(1, 8); +ADD_CASES(TC_ConsoleOut, + {{"^BM_Counters_kIsIterationInvariantRate/threads:%int " + "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); +ADD_CASES( + TC_JSONOut, + {{"\"name\": \"BM_Counters_kIsIterationInvariantRate/threads:%int\",$"}, + {"\"family_index\": 9,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Counters_kIsIterationInvariantRate/threads:%int\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": %int,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES( + TC_CSVOut, + {{"^\"BM_Counters_kIsIterationInvariantRate/threads:%int\",%csv_report," + "%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckIsIterationInvariantRate(Results const& e) { + double its = e.NumIterations(); + // this (and not real time) is the time used + double t = e.DurationCPUTime() / e.NumThreads(); + // check that the values are within 0.1% of the expected values + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, its * 1. * e.NumThreads() / t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, its * 2. * e.NumThreads() / t, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_kIsIterationInvariantRate/threads:%int", + &CheckIsIterationInvariantRate); +} // end namespace + +// ========================================================================= // +// --------------------- AvgIterations Counters Output --------------------- // +// ========================================================================= // + +namespace { +void BM_Counters_AvgIterations(benchmark::State& state) { + for (auto _ : state) { + } + namespace bm = benchmark; + state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterations}; + state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgIterations}; +} +BENCHMARK(BM_Counters_AvgIterations)->ThreadRange(1, 8); +ADD_CASES(TC_ConsoleOut, + {{"^BM_Counters_AvgIterations/threads:%int %console_report " + "bar=%hrfloat foo=%hrfloat$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_AvgIterations/threads:%int\",$"}, + {"\"family_index\": 10,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Counters_AvgIterations/threads:%int\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": %int,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_AvgIterations/" + "threads:%int\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckAvgIterations(Results const& e) { + double its = e.NumIterations(); + // check that the values are within 0.1% of the expected value + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. * e.NumThreads() / its, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. * e.NumThreads() / its, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_AvgIterations/threads:%int", + &CheckAvgIterations); +} // end namespace + +// ========================================================================= // +// ------------------- AvgIterationsRate Counters Output ------------------- // +// ========================================================================= // + +namespace { +void BM_Counters_kAvgIterationsRate(benchmark::State& state) { + for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + auto iterations = static_cast(state.iterations()) * + static_cast(state.iterations()); + benchmark::DoNotOptimize(iterations); + } + namespace bm = benchmark; + state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterationsRate}; + state.counters["bar"] = + bm::Counter{2, bm::Counter::kIsRate | bm::Counter::kAvgIterations}; +} +BENCHMARK(BM_Counters_kAvgIterationsRate)->ThreadRange(1, 8); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate/threads:%int " + "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_kAvgIterationsRate/threads:%int\",$"}, + {"\"family_index\": 11,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Counters_kAvgIterationsRate/threads:%int\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": %int,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Counters_kAvgIterationsRate/threads:%int\",%csv_report," + "%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckAvgIterationsRate(Results const& e) { + double its = e.NumIterations(); + // this (and not real time) is the time used + double t = e.DurationCPUTime() / e.NumThreads(); + // check that the values are within 0.1% of the expected values + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. * e.NumThreads() / its / t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. * e.NumThreads() / its / t, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_kAvgIterationsRate/threads:%int", + &CheckAvgIterationsRate); +} // end namespace + +// ========================================================================= // +// --------------------------- TEST CASES END ------------------------------ // +// ========================================================================= // + +int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + RunOutputTests(argc, argv); +} diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 5895883a2e..a49edb7f72 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -1,18 +1,20 @@ -load("@py_deps//:requirements.bzl", "requirement") +load("@rules_python//python:defs.bzl", "py_binary", "py_library") +load("@tools_pip_deps//:requirements.bzl", "requirement") py_library( name = "gbench", srcs = glob(["gbench/*.py"]), deps = [ - requirement("numpy"), - requirement("scipy"), + requirement("numpy"), + requirement("scipy"), ], ) py_binary( name = "compare", srcs = ["compare.py"], - python_version = "PY2", + imports = ["."], + python_version = "PY3", deps = [ ":gbench", ], diff --git a/tools/compare.py b/tools/compare.py index 01d2c89f50..1a656345c2 100755 --- a/tools/compare.py +++ b/tools/compare.py @@ -1,179 +1,246 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 + +# type: ignore -import unittest """ compare.py - versatile benchmark output compare tool """ import argparse -from argparse import ArgumentParser import json +import os import sys +import unittest +from argparse import ArgumentParser + import gbench -from gbench import util, report -from gbench.util import * +from gbench import report, util def check_inputs(in1, in2, flags): """ Perform checking on the user provided inputs and diagnose any abnormalities """ - in1_kind, in1_err = classify_input_file(in1) - in2_kind, in2_err = classify_input_file(in2) - output_file = find_benchmark_flag('--benchmark_out=', flags) - output_type = find_benchmark_flag('--benchmark_out_format=', flags) - if in1_kind == IT_Executable and in2_kind == IT_Executable and output_file: - print(("WARNING: '--benchmark_out=%s' will be passed to both " - "benchmarks causing it to be overwritten") % output_file) - if in1_kind == IT_JSON and in2_kind == IT_JSON and len(flags) > 0: - print("WARNING: passing optional flags has no effect since both " - "inputs are JSON") - if output_type is not None and output_type != 'json': - print(("ERROR: passing '--benchmark_out_format=%s' to 'compare.py`" - " is not supported.") % output_type) + in1_kind, _ = util.classify_input_file(in1) + in2_kind, _ = util.classify_input_file(in2) + output_file = util.find_benchmark_flag("--benchmark_out=", flags) + output_type = util.find_benchmark_flag("--benchmark_out_format=", flags) + if ( + in1_kind == util.IT_Executable + and in2_kind == util.IT_Executable + and output_file + ): + print( + ( + "WARNING: '--benchmark_out=%s' will be passed to both " + "benchmarks causing it to be overwritten" + ) + % output_file + ) + if in1_kind == util.IT_JSON and in2_kind == util.IT_JSON: + # When both sides are JSON the only supported flag is + # --benchmark_filter= + for flag in util.remove_benchmark_flags("--benchmark_filter=", flags): + print( + "WARNING: passing %s has no effect since both " + "inputs are JSON" % flag + ) + if output_type is not None and output_type != "json": + print( + ( + "ERROR: passing '--benchmark_out_format=%s' to 'compare.py`" + " is not supported." + ) + % output_type + ) sys.exit(1) def create_parser(): parser = ArgumentParser( - description='versatile benchmark output compare tool') + description="versatile benchmark output compare tool" + ) parser.add_argument( - '-a', - '--display_aggregates_only', - dest='display_aggregates_only', + "-a", + "--display_aggregates_only", + dest="display_aggregates_only", action="store_true", help="If there are repetitions, by default, we display everything - the" - " actual runs, and the aggregates computed. Sometimes, it is " - "desirable to only view the aggregates. E.g. when there are a lot " - "of repetitions. Do note that only the display is affected. " - "Internally, all the actual runs are still used, e.g. for U test.") + " actual runs, and the aggregates computed. Sometimes, it is " + "desirable to only view the aggregates. E.g. when there are a lot " + "of repetitions. Do note that only the display is affected. " + "Internally, all the actual runs are still used, e.g. for U test.", + ) parser.add_argument( - '--no-color', - dest='color', + "--no-color", + dest="color", default=True, action="store_false", - help="Do not use colors in the terminal output" + help="Do not use colors in the terminal output", ) parser.add_argument( - '-d', - '--dump_to_json', - dest='dump_to_json', - help="Additionally, dump benchmark comparison output to this file in JSON format.") + "-d", + "--dump_to_json", + dest="dump_to_json", + help=( + "Additionally, dump benchmark comparison output to this file in" + " JSON format." + ), + ) utest = parser.add_argument_group() utest.add_argument( - '--no-utest', - dest='utest', + "--no-utest", + dest="utest", default=True, action="store_false", - help="The tool can do a two-tailed Mann-Whitney U test with the null hypothesis that it is equally likely that a randomly selected value from one sample will be less than or greater than a randomly selected value from a second sample.\nWARNING: requires **LARGE** (no less than {}) number of repetitions to be meaningful!\nThe test is being done by default, if at least {} repetitions were done.\nThis option can disable the U Test.".format(report.UTEST_OPTIMAL_REPETITIONS, report.UTEST_MIN_REPETITIONS)) + help=( + "The tool can do a two-tailed Mann-Whitney U test with the null" + " hypothesis that it is equally likely that a randomly selected" + " value from one sample will be less than or greater than a" + " randomly selected value from a second sample.\nWARNING: requires" + f" **LARGE** (no less than {report.UTEST_OPTIMAL_REPETITIONS})" + " number of repetitions to be meaningful!\nThe test is being done" + f" by default, if at least {report.UTEST_MIN_REPETITIONS}" + " repetitions were done.\nThis option can disable the U Test." + ), + ) alpha_default = 0.05 utest.add_argument( "--alpha", - dest='utest_alpha', + dest="utest_alpha", default=alpha_default, type=float, - help=("significance level alpha. if the calculated p-value is below this value, then the result is said to be statistically significant and the null hypothesis is rejected.\n(default: %0.4f)") % - alpha_default) + help=( + "significance level alpha. if the calculated p-value is below this" + " value, then the result is said to be statistically significant" + " and the null hypothesis is rejected.\n(default: %0.4f)" + ) + % alpha_default, + ) subparsers = parser.add_subparsers( - help='This tool has multiple modes of operation:', - dest='mode') + help="This tool has multiple modes of operation:", dest="mode" + ) parser_a = subparsers.add_parser( - 'benchmarks', - help='The most simple use-case, compare all the output of these two benchmarks') - baseline = parser_a.add_argument_group( - 'baseline', 'The benchmark baseline') + "benchmarks", + help=( + "The most simple use-case, compare all the output of these two" + " benchmarks" + ), + ) + baseline = parser_a.add_argument_group("baseline", "The benchmark baseline") baseline.add_argument( - 'test_baseline', - metavar='test_baseline', - type=argparse.FileType('r'), + "test_baseline", + metavar="test_baseline", + type=argparse.FileType("r"), nargs=1, - help='A benchmark executable or JSON output file') + help="A benchmark executable or JSON output file", + ) contender = parser_a.add_argument_group( - 'contender', 'The benchmark that will be compared against the baseline') + "contender", "The benchmark that will be compared against the baseline" + ) contender.add_argument( - 'test_contender', - metavar='test_contender', - type=argparse.FileType('r'), + "test_contender", + metavar="test_contender", + type=argparse.FileType("r"), nargs=1, - help='A benchmark executable or JSON output file') + help="A benchmark executable or JSON output file", + ) parser_a.add_argument( - 'benchmark_options', - metavar='benchmark_options', + "benchmark_options", + metavar="benchmark_options", nargs=argparse.REMAINDER, - help='Arguments to pass when running benchmark executables') + help="Arguments to pass when running benchmark executables", + ) parser_b = subparsers.add_parser( - 'filters', help='Compare filter one with the filter two of benchmark') - baseline = parser_b.add_argument_group( - 'baseline', 'The benchmark baseline') + "filters", help="Compare filter one with the filter two of benchmark" + ) + baseline = parser_b.add_argument_group("baseline", "The benchmark baseline") baseline.add_argument( - 'test', - metavar='test', - type=argparse.FileType('r'), + "test", + metavar="test", + type=argparse.FileType("r"), nargs=1, - help='A benchmark executable or JSON output file') + help="A benchmark executable or JSON output file", + ) baseline.add_argument( - 'filter_baseline', - metavar='filter_baseline', + "filter_baseline", + metavar="filter_baseline", type=str, nargs=1, - help='The first filter, that will be used as baseline') + help="The first filter, that will be used as baseline", + ) contender = parser_b.add_argument_group( - 'contender', 'The benchmark that will be compared against the baseline') + "contender", "The benchmark that will be compared against the baseline" + ) contender.add_argument( - 'filter_contender', - metavar='filter_contender', + "filter_contender", + metavar="filter_contender", type=str, nargs=1, - help='The second filter, that will be compared against the baseline') + help="The second filter, that will be compared against the baseline", + ) parser_b.add_argument( - 'benchmark_options', - metavar='benchmark_options', + "benchmark_options", + metavar="benchmark_options", nargs=argparse.REMAINDER, - help='Arguments to pass when running benchmark executables') + help="Arguments to pass when running benchmark executables", + ) parser_c = subparsers.add_parser( - 'benchmarksfiltered', - help='Compare filter one of first benchmark with filter two of the second benchmark') - baseline = parser_c.add_argument_group( - 'baseline', 'The benchmark baseline') + "benchmarksfiltered", + help=( + "Compare filter one of first benchmark with filter two of the" + " second benchmark" + ), + ) + baseline = parser_c.add_argument_group("baseline", "The benchmark baseline") baseline.add_argument( - 'test_baseline', - metavar='test_baseline', - type=argparse.FileType('r'), + "test_baseline", + metavar="test_baseline", + type=argparse.FileType("r"), nargs=1, - help='A benchmark executable or JSON output file') + help="A benchmark executable or JSON output file", + ) baseline.add_argument( - 'filter_baseline', - metavar='filter_baseline', + "filter_baseline", + metavar="filter_baseline", type=str, nargs=1, - help='The first filter, that will be used as baseline') + help="The first filter, that will be used as baseline", + ) contender = parser_c.add_argument_group( - 'contender', 'The benchmark that will be compared against the baseline') + "contender", "The benchmark that will be compared against the baseline" + ) contender.add_argument( - 'test_contender', - metavar='test_contender', - type=argparse.FileType('r'), + "test_contender", + metavar="test_contender", + type=argparse.FileType("r"), nargs=1, - help='The second benchmark executable or JSON output file, that will be compared against the baseline') + help=( + "The second benchmark executable or JSON output file, that will be" + " compared against the baseline" + ), + ) contender.add_argument( - 'filter_contender', - metavar='filter_contender', + "filter_contender", + metavar="filter_contender", type=str, nargs=1, - help='The second filter, that will be compared against the baseline') + help="The second filter, that will be compared against the baseline", + ) parser_c.add_argument( - 'benchmark_options', - metavar='benchmark_options', + "benchmark_options", + metavar="benchmark_options", nargs=argparse.REMAINDER, - help='Arguments to pass when running benchmark executables') + help="Arguments to pass when running benchmark executables", + ) return parser @@ -188,16 +255,16 @@ def main(): assert not unknown_args benchmark_options = args.benchmark_options - if args.mode == 'benchmarks': + if args.mode == "benchmarks": test_baseline = args.test_baseline[0].name test_contender = args.test_contender[0].name - filter_baseline = '' - filter_contender = '' + filter_baseline = "" + filter_contender = "" # NOTE: if test_baseline == test_contender, you are analyzing the stdev - description = 'Comparing %s to %s' % (test_baseline, test_contender) - elif args.mode == 'filters': + description = "Comparing %s to %s" % (test_baseline, test_contender) + elif args.mode == "filters": test_baseline = args.test[0].name test_contender = args.test[0].name filter_baseline = args.filter_baseline[0] @@ -206,9 +273,12 @@ def main(): # NOTE: if filter_baseline == filter_contender, you are analyzing the # stdev - description = 'Comparing %s to %s (from %s)' % ( - filter_baseline, filter_contender, args.test[0].name) - elif args.mode == 'benchmarksfiltered': + description = "Comparing %s to %s (from %s)" % ( + filter_baseline, + filter_contender, + args.test[0].name, + ) + elif args.mode == "benchmarksfiltered": test_baseline = args.test_baseline[0].name test_contender = args.test_contender[0].name filter_baseline = args.filter_baseline[0] @@ -217,8 +287,12 @@ def main(): # NOTE: if test_baseline == test_contender and # filter_baseline == filter_contender, you are analyzing the stdev - description = 'Comparing %s (from %s) to %s (from %s)' % ( - filter_baseline, test_baseline, filter_contender, test_contender) + description = "Comparing %s (from %s) to %s (from %s)" % ( + filter_baseline, + test_baseline, + filter_contender, + test_contender, + ) else: # should never happen print("Unrecognized mode of operation: '%s'" % args.mode) @@ -228,199 +302,240 @@ def main(): check_inputs(test_baseline, test_contender, benchmark_options) if args.display_aggregates_only: - benchmark_options += ['--benchmark_display_aggregates_only=true'] + benchmark_options += ["--benchmark_display_aggregates_only=true"] options_baseline = [] options_contender = [] if filter_baseline and filter_contender: - options_baseline = ['--benchmark_filter=%s' % filter_baseline] - options_contender = ['--benchmark_filter=%s' % filter_contender] + options_baseline = ["--benchmark_filter=%s" % filter_baseline] + options_contender = ["--benchmark_filter=%s" % filter_contender] # Run the benchmarks and report the results - json1 = json1_orig = gbench.util.sort_benchmark_results(gbench.util.run_or_load_benchmark( - test_baseline, benchmark_options + options_baseline)) - json2 = json2_orig = gbench.util.sort_benchmark_results(gbench.util.run_or_load_benchmark( - test_contender, benchmark_options + options_contender)) + json1 = json1_orig = gbench.util.sort_benchmark_results( + gbench.util.run_or_load_benchmark( + test_baseline, benchmark_options + options_baseline + ) + ) + json2 = json2_orig = gbench.util.sort_benchmark_results( + gbench.util.run_or_load_benchmark( + test_contender, benchmark_options + options_contender + ) + ) # Now, filter the benchmarks so that the difference report can work if filter_baseline and filter_contender: - replacement = '[%s vs. %s]' % (filter_baseline, filter_contender) + replacement = "[%s vs. %s]" % (filter_baseline, filter_contender) json1 = gbench.report.filter_benchmark( - json1_orig, filter_baseline, replacement) + json1_orig, filter_baseline, replacement + ) json2 = gbench.report.filter_benchmark( - json2_orig, filter_contender, replacement) + json2_orig, filter_contender, replacement + ) - diff_report = gbench.report.get_difference_report( - json1, json2, args.utest) + diff_report = gbench.report.get_difference_report(json1, json2, args.utest) output_lines = gbench.report.print_difference_report( diff_report, args.display_aggregates_only, - args.utest, args.utest_alpha, args.color) + args.utest, + args.utest_alpha, + args.color, + ) print(description) for ln in output_lines: print(ln) # Optionally, diff and output to JSON if args.dump_to_json is not None: - with open(args.dump_to_json, 'w') as f_json: - json.dump(diff_report, f_json) + with open(args.dump_to_json, "w") as f_json: + json.dump(diff_report, f_json, indent=1) + class TestParser(unittest.TestCase): def setUp(self): self.parser = create_parser() testInputs = os.path.join( - os.path.dirname( - os.path.realpath(__file__)), - 'gbench', - 'Inputs') - self.testInput0 = os.path.join(testInputs, 'test1_run1.json') - self.testInput1 = os.path.join(testInputs, 'test1_run2.json') + os.path.dirname(os.path.realpath(__file__)), "gbench", "Inputs" + ) + self.testInput0 = os.path.join(testInputs, "test1_run1.json") + self.testInput1 = os.path.join(testInputs, "test1_run2.json") def test_benchmarks_basic(self): parsed = self.parser.parse_args( - ['benchmarks', self.testInput0, self.testInput1]) + ["benchmarks", self.testInput0, self.testInput1] + ) self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) - self.assertEqual(parsed.mode, 'benchmarks') + self.assertEqual(parsed.mode, "benchmarks") self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.test_contender[0].name, self.testInput1) self.assertFalse(parsed.benchmark_options) def test_benchmarks_basic_without_utest(self): parsed = self.parser.parse_args( - ['--no-utest', 'benchmarks', self.testInput0, self.testInput1]) + ["--no-utest", "benchmarks", self.testInput0, self.testInput1] + ) self.assertFalse(parsed.display_aggregates_only) self.assertFalse(parsed.utest) self.assertEqual(parsed.utest_alpha, 0.05) - self.assertEqual(parsed.mode, 'benchmarks') + self.assertEqual(parsed.mode, "benchmarks") self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.test_contender[0].name, self.testInput1) self.assertFalse(parsed.benchmark_options) def test_benchmarks_basic_display_aggregates_only(self): parsed = self.parser.parse_args( - ['-a', 'benchmarks', self.testInput0, self.testInput1]) + ["-a", "benchmarks", self.testInput0, self.testInput1] + ) self.assertTrue(parsed.display_aggregates_only) self.assertTrue(parsed.utest) - self.assertEqual(parsed.mode, 'benchmarks') + self.assertEqual(parsed.mode, "benchmarks") self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.test_contender[0].name, self.testInput1) self.assertFalse(parsed.benchmark_options) def test_benchmarks_basic_with_utest_alpha(self): parsed = self.parser.parse_args( - ['--alpha=0.314', 'benchmarks', self.testInput0, self.testInput1]) + ["--alpha=0.314", "benchmarks", self.testInput0, self.testInput1] + ) self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) self.assertEqual(parsed.utest_alpha, 0.314) - self.assertEqual(parsed.mode, 'benchmarks') + self.assertEqual(parsed.mode, "benchmarks") self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.test_contender[0].name, self.testInput1) self.assertFalse(parsed.benchmark_options) def test_benchmarks_basic_without_utest_with_utest_alpha(self): parsed = self.parser.parse_args( - ['--no-utest', '--alpha=0.314', 'benchmarks', self.testInput0, self.testInput1]) + [ + "--no-utest", + "--alpha=0.314", + "benchmarks", + self.testInput0, + self.testInput1, + ] + ) self.assertFalse(parsed.display_aggregates_only) self.assertFalse(parsed.utest) self.assertEqual(parsed.utest_alpha, 0.314) - self.assertEqual(parsed.mode, 'benchmarks') + self.assertEqual(parsed.mode, "benchmarks") self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.test_contender[0].name, self.testInput1) self.assertFalse(parsed.benchmark_options) def test_benchmarks_with_remainder(self): parsed = self.parser.parse_args( - ['benchmarks', self.testInput0, self.testInput1, 'd']) + ["benchmarks", self.testInput0, self.testInput1, "d"] + ) self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) - self.assertEqual(parsed.mode, 'benchmarks') + self.assertEqual(parsed.mode, "benchmarks") self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.test_contender[0].name, self.testInput1) - self.assertEqual(parsed.benchmark_options, ['d']) + self.assertEqual(parsed.benchmark_options, ["d"]) def test_benchmarks_with_remainder_after_doubleminus(self): parsed = self.parser.parse_args( - ['benchmarks', self.testInput0, self.testInput1, '--', 'e']) + ["benchmarks", self.testInput0, self.testInput1, "--", "e"] + ) self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) - self.assertEqual(parsed.mode, 'benchmarks') + self.assertEqual(parsed.mode, "benchmarks") self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.test_contender[0].name, self.testInput1) - self.assertEqual(parsed.benchmark_options, ['e']) + self.assertEqual(parsed.benchmark_options, ["e"]) def test_filters_basic(self): - parsed = self.parser.parse_args( - ['filters', self.testInput0, 'c', 'd']) + parsed = self.parser.parse_args(["filters", self.testInput0, "c", "d"]) self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) - self.assertEqual(parsed.mode, 'filters') + self.assertEqual(parsed.mode, "filters") self.assertEqual(parsed.test[0].name, self.testInput0) - self.assertEqual(parsed.filter_baseline[0], 'c') - self.assertEqual(parsed.filter_contender[0], 'd') + self.assertEqual(parsed.filter_baseline[0], "c") + self.assertEqual(parsed.filter_contender[0], "d") self.assertFalse(parsed.benchmark_options) def test_filters_with_remainder(self): parsed = self.parser.parse_args( - ['filters', self.testInput0, 'c', 'd', 'e']) + ["filters", self.testInput0, "c", "d", "e"] + ) self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) - self.assertEqual(parsed.mode, 'filters') + self.assertEqual(parsed.mode, "filters") self.assertEqual(parsed.test[0].name, self.testInput0) - self.assertEqual(parsed.filter_baseline[0], 'c') - self.assertEqual(parsed.filter_contender[0], 'd') - self.assertEqual(parsed.benchmark_options, ['e']) + self.assertEqual(parsed.filter_baseline[0], "c") + self.assertEqual(parsed.filter_contender[0], "d") + self.assertEqual(parsed.benchmark_options, ["e"]) def test_filters_with_remainder_after_doubleminus(self): parsed = self.parser.parse_args( - ['filters', self.testInput0, 'c', 'd', '--', 'f']) + ["filters", self.testInput0, "c", "d", "--", "f"] + ) self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) - self.assertEqual(parsed.mode, 'filters') + self.assertEqual(parsed.mode, "filters") self.assertEqual(parsed.test[0].name, self.testInput0) - self.assertEqual(parsed.filter_baseline[0], 'c') - self.assertEqual(parsed.filter_contender[0], 'd') - self.assertEqual(parsed.benchmark_options, ['f']) + self.assertEqual(parsed.filter_baseline[0], "c") + self.assertEqual(parsed.filter_contender[0], "d") + self.assertEqual(parsed.benchmark_options, ["f"]) def test_benchmarksfiltered_basic(self): parsed = self.parser.parse_args( - ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e']) + ["benchmarksfiltered", self.testInput0, "c", self.testInput1, "e"] + ) self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) - self.assertEqual(parsed.mode, 'benchmarksfiltered') + self.assertEqual(parsed.mode, "benchmarksfiltered") self.assertEqual(parsed.test_baseline[0].name, self.testInput0) - self.assertEqual(parsed.filter_baseline[0], 'c') + self.assertEqual(parsed.filter_baseline[0], "c") self.assertEqual(parsed.test_contender[0].name, self.testInput1) - self.assertEqual(parsed.filter_contender[0], 'e') + self.assertEqual(parsed.filter_contender[0], "e") self.assertFalse(parsed.benchmark_options) def test_benchmarksfiltered_with_remainder(self): parsed = self.parser.parse_args( - ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', 'f']) + [ + "benchmarksfiltered", + self.testInput0, + "c", + self.testInput1, + "e", + "f", + ] + ) self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) - self.assertEqual(parsed.mode, 'benchmarksfiltered') + self.assertEqual(parsed.mode, "benchmarksfiltered") self.assertEqual(parsed.test_baseline[0].name, self.testInput0) - self.assertEqual(parsed.filter_baseline[0], 'c') + self.assertEqual(parsed.filter_baseline[0], "c") self.assertEqual(parsed.test_contender[0].name, self.testInput1) - self.assertEqual(parsed.filter_contender[0], 'e') - self.assertEqual(parsed.benchmark_options[0], 'f') + self.assertEqual(parsed.filter_contender[0], "e") + self.assertEqual(parsed.benchmark_options[0], "f") def test_benchmarksfiltered_with_remainder_after_doubleminus(self): parsed = self.parser.parse_args( - ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', '--', 'g']) + [ + "benchmarksfiltered", + self.testInput0, + "c", + self.testInput1, + "e", + "--", + "g", + ] + ) self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) - self.assertEqual(parsed.mode, 'benchmarksfiltered') + self.assertEqual(parsed.mode, "benchmarksfiltered") self.assertEqual(parsed.test_baseline[0].name, self.testInput0) - self.assertEqual(parsed.filter_baseline[0], 'c') + self.assertEqual(parsed.filter_baseline[0], "c") self.assertEqual(parsed.test_contender[0].name, self.testInput1) - self.assertEqual(parsed.filter_contender[0], 'e') - self.assertEqual(parsed.benchmark_options[0], 'g') + self.assertEqual(parsed.filter_contender[0], "e") + self.assertEqual(parsed.benchmark_options[0], "g") -if __name__ == '__main__': +if __name__ == "__main__": # unittest.main() main() diff --git a/tools/gbench/Inputs/test5_run0.json b/tools/gbench/Inputs/test5_run0.json new file mode 100644 index 0000000000..074103b11d --- /dev/null +++ b/tools/gbench/Inputs/test5_run0.json @@ -0,0 +1,18 @@ +{ + "context": { + "date": "2016-08-02 17:44:46", + "num_cpus": 4, + "mhz_per_cpu": 4228, + "cpu_scaling_enabled": false, + "library_build_type": "release" + }, + "benchmarks": [ + { + "name": "BM_ManyRepetitions", + "iterations": 1000, + "real_time": 1, + "cpu_time": 1000, + "time_unit": "s" + } + ] +} diff --git a/tools/gbench/Inputs/test5_run1.json b/tools/gbench/Inputs/test5_run1.json new file mode 100644 index 0000000000..430df9f0da --- /dev/null +++ b/tools/gbench/Inputs/test5_run1.json @@ -0,0 +1,18 @@ +{ + "context": { + "date": "2016-08-02 17:44:46", + "num_cpus": 4, + "mhz_per_cpu": 4228, + "cpu_scaling_enabled": false, + "library_build_type": "release" + }, + "benchmarks": [ + { + "name": "BM_ManyRepetitions", + "iterations": 1000, + "real_time": 1000, + "cpu_time": 1, + "time_unit": "s" + } + ] +} diff --git a/tools/gbench/__init__.py b/tools/gbench/__init__.py index fce1a1acfb..9212568814 100644 --- a/tools/gbench/__init__.py +++ b/tools/gbench/__init__.py @@ -1,8 +1,8 @@ """Google Benchmark tooling""" -__author__ = 'Eric Fiselier' -__email__ = 'eric@efcs.ca' +__author__ = "Eric Fiselier" +__email__ = "eric@efcs.ca" __versioninfo__ = (0, 5, 0) -__version__ = '.'.join(str(v) for v in __versioninfo__) + 'dev' +__version__ = ".".join(str(v) for v in __versioninfo__) + "dev" -__all__ = [] +__all__ = [] # type: ignore diff --git a/tools/gbench/report.py b/tools/gbench/report.py index 4f2ea3ef5e..e143e45a71 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -1,48 +1,56 @@ -"""report.py - Utilities for reporting statistics about benchmark results +# type: ignore + +""" +report.py - Utilities for reporting statistics about benchmark results """ -import unittest -import os -import re import copy +import os import random +import re +import unittest -from scipy.stats import mannwhitneyu, gmean from numpy import array -from pandas import Timedelta +from scipy.stats import gmean, mannwhitneyu -class BenchmarkColor(object): +class BenchmarkColor: def __init__(self, name, code): self.name = name self.code = code def __repr__(self): - return '%s%r' % (self.__class__.__name__, - (self.name, self.code)) + return "%s%r" % (self.__class__.__name__, (self.name, self.code)) def __format__(self, format): return self.code # Benchmark Colors Enumeration -BC_NONE = BenchmarkColor('NONE', '') -BC_MAGENTA = BenchmarkColor('MAGENTA', '\033[95m') -BC_CYAN = BenchmarkColor('CYAN', '\033[96m') -BC_OKBLUE = BenchmarkColor('OKBLUE', '\033[94m') -BC_OKGREEN = BenchmarkColor('OKGREEN', '\033[32m') -BC_HEADER = BenchmarkColor('HEADER', '\033[92m') -BC_WARNING = BenchmarkColor('WARNING', '\033[93m') -BC_WHITE = BenchmarkColor('WHITE', '\033[97m') -BC_FAIL = BenchmarkColor('FAIL', '\033[91m') -BC_ENDC = BenchmarkColor('ENDC', '\033[0m') -BC_BOLD = BenchmarkColor('BOLD', '\033[1m') -BC_UNDERLINE = BenchmarkColor('UNDERLINE', '\033[4m') +BC_NONE = BenchmarkColor("NONE", "") +BC_MAGENTA = BenchmarkColor("MAGENTA", "\033[95m") +BC_CYAN = BenchmarkColor("CYAN", "\033[96m") +BC_OKBLUE = BenchmarkColor("OKBLUE", "\033[94m") +BC_OKGREEN = BenchmarkColor("OKGREEN", "\033[32m") +BC_HEADER = BenchmarkColor("HEADER", "\033[92m") +BC_WARNING = BenchmarkColor("WARNING", "\033[93m") +BC_WHITE = BenchmarkColor("WHITE", "\033[97m") +BC_FAIL = BenchmarkColor("FAIL", "\033[91m") +BC_ENDC = BenchmarkColor("ENDC", "\033[0m") +BC_BOLD = BenchmarkColor("BOLD", "\033[1m") +BC_UNDERLINE = BenchmarkColor("UNDERLINE", "\033[4m") UTEST_MIN_REPETITIONS = 2 UTEST_OPTIMAL_REPETITIONS = 9 # Lowest reasonable number, More is better. UTEST_COL_NAME = "_pvalue" +_TIME_UNIT_TO_SECONDS_MULTIPLIER = { + "s": 1.0, + "ms": 1e-3, + "us": 1e-6, + "ns": 1e-9, +} + def color_format(use_color, fmt_str, *args, **kwargs): """ @@ -53,10 +61,14 @@ def color_format(use_color, fmt_str, *args, **kwargs): """ assert use_color is True or use_color is False if not use_color: - args = [arg if not isinstance(arg, BenchmarkColor) else BC_NONE - for arg in args] - kwargs = {key: arg if not isinstance(arg, BenchmarkColor) else BC_NONE - for key, arg in kwargs.items()} + args = [ + arg if not isinstance(arg, BenchmarkColor) else BC_NONE + for arg in args + ] + kwargs = { + key: arg if not isinstance(arg, BenchmarkColor) else BC_NONE + for key, arg in kwargs.items() + } return fmt_str.format(*args, **kwargs) @@ -67,8 +79,8 @@ def find_longest_name(benchmark_list): """ longest_name = 1 for bc in benchmark_list: - if len(bc['name']) > longest_name: - longest_name = len(bc['name']) + if len(bc["name"]) > longest_name: + longest_name = len(bc["name"]) return longest_name @@ -89,13 +101,13 @@ def filter_benchmark(json_orig, family, replacement=""): """ regex = re.compile(family) filtered = {} - filtered['benchmarks'] = [] - for be in json_orig['benchmarks']: - if not regex.search(be['name']): + filtered["benchmarks"] = [] + for be in json_orig["benchmarks"]: + if not regex.search(be["name"]): continue filteredbench = copy.deepcopy(be) # Do NOT modify the old name! - filteredbench['name'] = regex.sub(replacement, filteredbench['name']) - filtered['benchmarks'].append(filteredbench) + filteredbench["name"] = regex.sub(replacement, filteredbench["name"]) + filtered["benchmarks"].append(filteredbench) return filtered @@ -104,9 +116,11 @@ def get_unique_benchmark_names(json): While *keeping* the order, give all the unique 'names' used for benchmarks. """ seen = set() - uniqued = [x['name'] for x in json['benchmarks'] - if x['name'] not in seen and - (seen.add(x['name']) or True)] + uniqued = [ + x["name"] + for x in json["benchmarks"] + if x["name"] not in seen and (seen.add(x["name"]) or True) + ] return uniqued @@ -119,7 +133,7 @@ def intersect(list1, list2): def is_potentially_comparable_benchmark(x): - return ('time_unit' in x and 'real_time' in x and 'cpu_time' in x) + return "time_unit" in x and "real_time" in x and "cpu_time" in x def partition_benchmarks(json1, json2): @@ -136,18 +150,24 @@ def partition_benchmarks(json1, json2): time_unit = None # Pick the time unit from the first entry of the lhs benchmark. # We should be careful not to crash with unexpected input. - for x in json1['benchmarks']: - if (x['name'] == name and is_potentially_comparable_benchmark(x)): - time_unit = x['time_unit'] + for x in json1["benchmarks"]: + if x["name"] == name and is_potentially_comparable_benchmark(x): + time_unit = x["time_unit"] break if time_unit is None: continue # Filter by name and time unit. # All the repetitions are assumed to be comparable. - lhs = [x for x in json1['benchmarks'] if x['name'] == name and - x['time_unit'] == time_unit] - rhs = [x for x in json2['benchmarks'] if x['name'] == name and - x['time_unit'] == time_unit] + lhs = [ + x + for x in json1["benchmarks"] + if x["name"] == name and x["time_unit"] == time_unit + ] + rhs = [ + x + for x in json2["benchmarks"] + if x["name"] == name and x["time_unit"] == time_unit + ] partitions.append([lhs, rhs]) return partitions @@ -157,9 +177,9 @@ def get_timedelta_field_as_seconds(benchmark, field_name): Get value of field_name field of benchmark, which is time with time unit time_unit, as time in seconds. """ - time_unit = benchmark['time_unit'] if 'time_unit' in benchmark else 's' - dt = Timedelta(benchmark[field_name], time_unit) - return dt / Timedelta(1, 's') + timedelta = benchmark[field_name] + time_unit = benchmark.get("time_unit", "s") + return timedelta * _TIME_UNIT_TO_SECONDS_MULTIPLIER.get(time_unit) def calculate_geomean(json): @@ -168,11 +188,15 @@ def calculate_geomean(json): and calculate their geomean. """ times = [] - for benchmark in json['benchmarks']: - if 'run_type' in benchmark and benchmark['run_type'] == 'aggregate': + for benchmark in json["benchmarks"]: + if "run_type" in benchmark and benchmark["run_type"] == "aggregate": continue - times.append([get_timedelta_field_as_seconds(benchmark, 'real_time'), - get_timedelta_field_as_seconds(benchmark, 'cpu_time')]) + times.append( + [ + get_timedelta_field_as_seconds(benchmark, "real_time"), + get_timedelta_field_as_seconds(benchmark, "cpu_time"), + ] + ) return gmean(times) if times else array([]) @@ -184,19 +208,23 @@ def extract_field(partition, field_name): def calc_utest(timings_cpu, timings_time): - min_rep_cnt = min(len(timings_time[0]), - len(timings_time[1]), - len(timings_cpu[0]), - len(timings_cpu[1])) + min_rep_cnt = min( + len(timings_time[0]), + len(timings_time[1]), + len(timings_cpu[0]), + len(timings_cpu[1]), + ) # Does *everything* has at least UTEST_MIN_REPETITIONS repetitions? if min_rep_cnt < UTEST_MIN_REPETITIONS: return False, None, None time_pvalue = mannwhitneyu( - timings_time[0], timings_time[1], alternative='two-sided').pvalue + timings_time[0], timings_time[1], alternative="two-sided" + ).pvalue cpu_pvalue = mannwhitneyu( - timings_cpu[0], timings_cpu[1], alternative='two-sided').pvalue + timings_cpu[0], timings_cpu[1], alternative="two-sided" + ).pvalue return (min_rep_cnt >= UTEST_OPTIMAL_REPETITIONS), cpu_pvalue, time_pvalue @@ -206,38 +234,47 @@ def get_utest_color(pval): return BC_FAIL if pval >= utest_alpha else BC_OKGREEN # Check if we failed miserably with minimum required repetitions for utest - if not utest['have_optimal_repetitions'] and utest['cpu_pvalue'] is None and utest['time_pvalue'] is None: + if ( + not utest["have_optimal_repetitions"] + and utest["cpu_pvalue"] is None + and utest["time_pvalue"] is None + ): return [] dsc = "U Test, Repetitions: {} vs {}".format( - utest['nr_of_repetitions'], utest['nr_of_repetitions_other']) + utest["nr_of_repetitions"], utest["nr_of_repetitions_other"] + ) dsc_color = BC_OKGREEN # We still got some results to show but issue a warning about it. - if not utest['have_optimal_repetitions']: + if not utest["have_optimal_repetitions"]: dsc_color = BC_WARNING - dsc += ". WARNING: Results unreliable! {}+ repetitions recommended.".format( - UTEST_OPTIMAL_REPETITIONS) + dsc += ( + f". WARNING: Results unreliable! {UTEST_OPTIMAL_REPETITIONS}+" + " repetitions recommended." + ) special_str = "{}{:<{}s}{endc}{}{:16.4f}{endc}{}{:16.4f}{endc}{} {}" - return [color_format(use_color, - special_str, - BC_HEADER, - "{}{}".format(bc_name, UTEST_COL_NAME), - first_col_width, - get_utest_color( - utest['time_pvalue']), utest['time_pvalue'], - get_utest_color( - utest['cpu_pvalue']), utest['cpu_pvalue'], - dsc_color, dsc, - endc=BC_ENDC)] - - -def get_difference_report( - json1, - json2, - utest=False): + return [ + color_format( + use_color, + special_str, + BC_HEADER, + f"{bc_name}{UTEST_COL_NAME}", + first_col_width, + get_utest_color(utest["time_pvalue"]), + utest["time_pvalue"], + get_utest_color(utest["cpu_pvalue"]), + utest["cpu_pvalue"], + dsc_color, + dsc, + endc=BC_ENDC, + ) + ] + + +def get_difference_report(json1, json2, utest=False): """ Calculate and report the difference between each test of two benchmarks runs specified as 'json1' and 'json2'. Output is another json containing @@ -248,37 +285,44 @@ def get_difference_report( diff_report = [] partitions = partition_benchmarks(json1, json2) for partition in partitions: - benchmark_name = partition[0][0]['name'] - label = partition[0][0]['label'] if 'label' in partition[0][0] else '' - time_unit = partition[0][0]['time_unit'] + benchmark_name = partition[0][0]["name"] + label = partition[0][0].get("label", "") + time_unit = partition[0][0]["time_unit"] measurements = [] utest_results = {} # Careful, we may have different repetition count. for i in range(min(len(partition[0]), len(partition[1]))): bn = partition[0][i] other_bench = partition[1][i] - measurements.append({ - 'real_time': bn['real_time'], - 'cpu_time': bn['cpu_time'], - 'real_time_other': other_bench['real_time'], - 'cpu_time_other': other_bench['cpu_time'], - 'time': calculate_change(bn['real_time'], other_bench['real_time']), - 'cpu': calculate_change(bn['cpu_time'], other_bench['cpu_time']) - }) + measurements.append( + { + "real_time": bn["real_time"], + "cpu_time": bn["cpu_time"], + "real_time_other": other_bench["real_time"], + "cpu_time_other": other_bench["cpu_time"], + "time": calculate_change( + bn["real_time"], other_bench["real_time"] + ), + "cpu": calculate_change( + bn["cpu_time"], other_bench["cpu_time"] + ), + } + ) # After processing the whole partition, if requested, do the U test. if utest: - timings_cpu = extract_field(partition, 'cpu_time') - timings_time = extract_field(partition, 'real_time') + timings_cpu = extract_field(partition, "cpu_time") + timings_time = extract_field(partition, "real_time") have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest( - timings_cpu, timings_time) - if cpu_pvalue and time_pvalue: + timings_cpu, timings_time + ) + if cpu_pvalue is not None and time_pvalue is not None: utest_results = { - 'have_optimal_repetitions': have_optimal_repetitions, - 'cpu_pvalue': cpu_pvalue, - 'time_pvalue': time_pvalue, - 'nr_of_repetitions': len(timings_cpu[0]), - 'nr_of_repetitions_other': len(timings_cpu[1]) + "have_optimal_repetitions": have_optimal_repetitions, + "cpu_pvalue": cpu_pvalue, + "time_pvalue": time_pvalue, + "nr_of_repetitions": len(timings_cpu[0]), + "nr_of_repetitions_other": len(timings_cpu[1]), } # Store only if we had any measurements for given benchmark. @@ -286,47 +330,59 @@ def get_difference_report( # time units which are not compatible with other time units in the # benchmark suite. if measurements: - run_type = partition[0][0]['run_type'] if 'run_type' in partition[0][0] else '' - aggregate_name = partition[0][0]['aggregate_name'] if run_type == 'aggregate' and 'aggregate_name' in partition[0][0] else '' - diff_report.append({ - 'name': benchmark_name, - 'label': label, - 'measurements': measurements, - 'time_unit': time_unit, - 'run_type': run_type, - 'aggregate_name': aggregate_name, - 'utest': utest_results - }) + run_type = partition[0][0].get("run_type", "") + aggregate_name = ( + partition[0][0]["aggregate_name"] + if run_type == "aggregate" + and "aggregate_name" in partition[0][0] + else "" + ) + diff_report.append( + { + "name": benchmark_name, + "label": label, + "measurements": measurements, + "time_unit": time_unit, + "run_type": run_type, + "aggregate_name": aggregate_name, + "utest": utest_results, + } + ) lhs_gmean = calculate_geomean(json1) rhs_gmean = calculate_geomean(json2) if lhs_gmean.any() and rhs_gmean.any(): - diff_report.append({ - 'name': 'OVERALL_GEOMEAN', - 'label': '', - 'measurements': [{ - 'real_time': lhs_gmean[0], - 'cpu_time': lhs_gmean[1], - 'real_time_other': rhs_gmean[0], - 'cpu_time_other': rhs_gmean[1], - 'time': calculate_change(lhs_gmean[0], rhs_gmean[0]), - 'cpu': calculate_change(lhs_gmean[1], rhs_gmean[1]) - }], - 'time_unit': 's', - 'run_type': 'aggregate', - 'aggregate_name': 'geomean', - 'utest': {} - }) + diff_report.append( + { + "name": "OVERALL_GEOMEAN", + "label": "", + "measurements": [ + { + "real_time": lhs_gmean[0], + "cpu_time": lhs_gmean[1], + "real_time_other": rhs_gmean[0], + "cpu_time_other": rhs_gmean[1], + "time": calculate_change(lhs_gmean[0], rhs_gmean[0]), + "cpu": calculate_change(lhs_gmean[1], rhs_gmean[1]), + } + ], + "time_unit": "s", + "run_type": "aggregate", + "aggregate_name": "geomean", + "utest": {}, + } + ) return diff_report def print_difference_report( - json_diff_report, - include_aggregates_only=False, - utest=False, - utest_alpha=0.05, - use_color=True): + json_diff_report, + include_aggregates_only=False, + utest=False, + utest_alpha=0.05, + use_color=True, +): """ Calculate and report the difference between each test of two benchmarks runs specified as 'json1' and 'json2'. @@ -342,44 +398,58 @@ def get_color(res): return BC_CYAN first_col_width = find_longest_name(json_diff_report) - first_col_width = max( - first_col_width, - len('Benchmark')) + first_col_width = max(first_col_width, len("Benchmark")) first_col_width += len(UTEST_COL_NAME) - first_line = "{:<{}s}Time CPU Time Old Time New CPU Old CPU New".format( - 'Benchmark', 12 + first_col_width) - output_strs = [first_line, '-' * len(first_line)] - - fmt_str = "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}{endc}{:14.0f}{:14.0f}" + fmt_str = ( + "{:<{}s}Time CPU Time Old Time New CPU Old" + " CPU New" + ) + first_line = fmt_str.format("Benchmark", 12 + first_col_width) + output_strs = [first_line, "-" * len(first_line)] + + fmt_str = ( + "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}" + "{endc}{:14.0f}{:14.0f}" + ) for benchmark in json_diff_report: # *If* we were asked to only include aggregates, # and if it is non-aggregate, then don't print it. - if not include_aggregates_only or not 'run_type' in benchmark or benchmark['run_type'] == 'aggregate': - for measurement in benchmark['measurements']: - output_strs += [color_format(use_color, - fmt_str, - BC_HEADER, - benchmark['name'], - first_col_width, - get_color(measurement['time']), - measurement['time'], - get_color(measurement['cpu']), - measurement['cpu'], - measurement['real_time'], - measurement['real_time_other'], - measurement['cpu_time'], - measurement['cpu_time_other'], - endc=BC_ENDC)] + if ( + not include_aggregates_only + or "run_type" not in benchmark + or benchmark["run_type"] == "aggregate" + ): + for measurement in benchmark["measurements"]: + output_strs += [ + color_format( + use_color, + fmt_str, + BC_HEADER, + benchmark["name"], + first_col_width, + get_color(measurement["time"]), + measurement["time"], + get_color(measurement["cpu"]), + measurement["cpu"], + measurement["real_time"], + measurement["real_time_other"], + measurement["cpu_time"], + measurement["cpu_time_other"], + endc=BC_ENDC, + ) + ] # After processing the measurements, if requested and # if applicable (e.g. u-test exists for given benchmark), # print the U test. - if utest and benchmark['utest']: - output_strs += print_utest(benchmark['name'], - benchmark['utest'], - utest_alpha=utest_alpha, - first_col_width=first_col_width, - use_color=use_color) + if utest and benchmark["utest"]: + output_strs += print_utest( + benchmark["name"], + benchmark["utest"], + utest_alpha=utest_alpha, + first_col_width=first_col_width, + use_color=use_color, + ) return output_strs @@ -391,29 +461,29 @@ def get_color(res): class TestGetUniqueBenchmarkNames(unittest.TestCase): def load_results(self): import json + testInputs = os.path.join( - os.path.dirname( - os.path.realpath(__file__)), - 'Inputs') - testOutput = os.path.join(testInputs, 'test3_run0.json') - with open(testOutput, 'r') as f: + os.path.dirname(os.path.realpath(__file__)), "Inputs" + ) + testOutput = os.path.join(testInputs, "test3_run0.json") + with open(testOutput) as f: json = json.load(f) return json def test_basic(self): expect_lines = [ - 'BM_One', - 'BM_Two', - 'short', # These two are not sorted - 'medium', # These two are not sorted + "BM_One", + "BM_Two", + "short", # These two are not sorted + "medium", # These two are not sorted ] json = self.load_results() output_lines = get_unique_benchmark_names(json) print("\n") print("\n".join(output_lines)) self.assertEqual(len(output_lines), len(expect_lines)) - for i in range(0, len(output_lines)): - self.assertEqual(expect_lines[i], output_lines[i]) + for i, output_line in enumerate(output_lines): + self.assertEqual(expect_lines[i], output_line) class TestReportDifference(unittest.TestCase): @@ -421,15 +491,15 @@ class TestReportDifference(unittest.TestCase): def setUpClass(cls): def load_results(): import json + testInputs = os.path.join( - os.path.dirname( - os.path.realpath(__file__)), - 'Inputs') - testOutput1 = os.path.join(testInputs, 'test1_run1.json') - testOutput2 = os.path.join(testInputs, 'test1_run2.json') - with open(testOutput1, 'r') as f: + os.path.dirname(os.path.realpath(__file__)), "Inputs" + ) + testOutput1 = os.path.join(testInputs, "test1_run1.json") + testOutput2 = os.path.join(testInputs, "test1_run2.json") + with open(testOutput1) as f: json1 = json.load(f) - with open(testOutput2, 'r') as f: + with open(testOutput2) as f: json2 = json.load(f) return json1, json2 @@ -438,171 +508,325 @@ def load_results(): def test_json_diff_report_pretty_printing(self): expect_lines = [ - ['BM_SameTimes', '+0.0000', '+0.0000', '10', '10', '10', '10'], - ['BM_2xFaster', '-0.5000', '-0.5000', '50', '25', '50', '25'], - ['BM_2xSlower', '+1.0000', '+1.0000', '50', '100', '50', '100'], - ['BM_1PercentFaster', '-0.0100', '-0.0100', '100', '99', '100', '99'], - ['BM_1PercentSlower', '+0.0100', '+0.0100', '100', '101', '100', '101'], - ['BM_10PercentFaster', '-0.1000', '-0.1000', '100', '90', '100', '90'], - ['BM_10PercentSlower', '+0.1000', '+0.1000', '100', '110', '100', '110'], - ['BM_100xSlower', '+99.0000', '+99.0000', - '100', '10000', '100', '10000'], - ['BM_100xFaster', '-0.9900', '-0.9900', - '10000', '100', '10000', '100'], - ['BM_10PercentCPUToTime', '+0.1000', - '-0.1000', '100', '110', '100', '90'], - ['BM_ThirdFaster', '-0.3333', '-0.3334', '100', '67', '100', '67'], - ['BM_NotBadTimeUnit', '-0.9000', '+0.2000', '0', '0', '0', '1'], - ['BM_hasLabel', '+0.0000', '+0.0000', '1', '1', '1', '1'], - ['OVERALL_GEOMEAN', '-0.8117', '-0.7783', '0', '0', '0', '0'] + ["BM_SameTimes", "+0.0000", "+0.0000", "10", "10", "10", "10"], + ["BM_2xFaster", "-0.5000", "-0.5000", "50", "25", "50", "25"], + ["BM_2xSlower", "+1.0000", "+1.0000", "50", "100", "50", "100"], + [ + "BM_1PercentFaster", + "-0.0100", + "-0.0100", + "100", + "99", + "100", + "99", + ], + [ + "BM_1PercentSlower", + "+0.0100", + "+0.0100", + "100", + "101", + "100", + "101", + ], + [ + "BM_10PercentFaster", + "-0.1000", + "-0.1000", + "100", + "90", + "100", + "90", + ], + [ + "BM_10PercentSlower", + "+0.1000", + "+0.1000", + "100", + "110", + "100", + "110", + ], + [ + "BM_100xSlower", + "+99.0000", + "+99.0000", + "100", + "10000", + "100", + "10000", + ], + [ + "BM_100xFaster", + "-0.9900", + "-0.9900", + "10000", + "100", + "10000", + "100", + ], + [ + "BM_10PercentCPUToTime", + "+0.1000", + "-0.1000", + "100", + "110", + "100", + "90", + ], + ["BM_ThirdFaster", "-0.3333", "-0.3334", "100", "67", "100", "67"], + ["BM_NotBadTimeUnit", "-0.9000", "+0.2000", "0", "0", "0", "1"], + ["BM_hasLabel", "+0.0000", "+0.0000", "1", "1", "1", "1"], + ["OVERALL_GEOMEAN", "-0.8113", "-0.7779", "0", "0", "0", "0"], ] output_lines_with_header = print_difference_report( - self.json_diff_report, use_color=False) + self.json_diff_report, use_color=False + ) output_lines = output_lines_with_header[2:] print("\n") print("\n".join(output_lines_with_header)) self.assertEqual(len(output_lines), len(expect_lines)) - for i in range(0, len(output_lines)): - parts = [x for x in output_lines[i].split(' ') if x] + for i, output_line in enumerate(output_lines): + parts = [x for x in output_line.split(" ") if x] self.assertEqual(len(parts), 7) self.assertEqual(expect_lines[i], parts) def test_json_diff_report_output(self): expected_output = [ { - 'name': 'BM_SameTimes', - 'label': '', - 'measurements': [{'time': 0.0000, 'cpu': 0.0000, - 'real_time': 10, 'real_time_other': 10, - 'cpu_time': 10, 'cpu_time_other': 10}], - 'time_unit': 'ns', - 'utest': {} + "name": "BM_SameTimes", + "label": "", + "measurements": [ + { + "time": 0.0000, + "cpu": 0.0000, + "real_time": 10, + "real_time_other": 10, + "cpu_time": 10, + "cpu_time_other": 10, + } + ], + "time_unit": "ns", + "utest": {}, }, { - 'name': 'BM_2xFaster', - 'label': '', - 'measurements': [{'time': -0.5000, 'cpu': -0.5000, - 'real_time': 50, 'real_time_other': 25, - 'cpu_time': 50, 'cpu_time_other': 25}], - 'time_unit': 'ns', - 'utest': {} + "name": "BM_2xFaster", + "label": "", + "measurements": [ + { + "time": -0.5000, + "cpu": -0.5000, + "real_time": 50, + "real_time_other": 25, + "cpu_time": 50, + "cpu_time_other": 25, + } + ], + "time_unit": "ns", + "utest": {}, }, { - 'name': 'BM_2xSlower', - 'label': '', - 'measurements': [{'time': 1.0000, 'cpu': 1.0000, - 'real_time': 50, 'real_time_other': 100, - 'cpu_time': 50, 'cpu_time_other': 100}], - 'time_unit': 'ns', - 'utest': {} + "name": "BM_2xSlower", + "label": "", + "measurements": [ + { + "time": 1.0000, + "cpu": 1.0000, + "real_time": 50, + "real_time_other": 100, + "cpu_time": 50, + "cpu_time_other": 100, + } + ], + "time_unit": "ns", + "utest": {}, }, { - 'name': 'BM_1PercentFaster', - 'label': '', - 'measurements': [{'time': -0.0100, 'cpu': -0.0100, - 'real_time': 100, 'real_time_other': 98.9999999, - 'cpu_time': 100, 'cpu_time_other': 98.9999999}], - 'time_unit': 'ns', - 'utest': {} + "name": "BM_1PercentFaster", + "label": "", + "measurements": [ + { + "time": -0.0100, + "cpu": -0.0100, + "real_time": 100, + "real_time_other": 98.9999999, + "cpu_time": 100, + "cpu_time_other": 98.9999999, + } + ], + "time_unit": "ns", + "utest": {}, }, { - 'name': 'BM_1PercentSlower', - 'label': '', - 'measurements': [{'time': 0.0100, 'cpu': 0.0100, - 'real_time': 100, 'real_time_other': 101, - 'cpu_time': 100, 'cpu_time_other': 101}], - 'time_unit': 'ns', - 'utest': {} + "name": "BM_1PercentSlower", + "label": "", + "measurements": [ + { + "time": 0.0100, + "cpu": 0.0100, + "real_time": 100, + "real_time_other": 101, + "cpu_time": 100, + "cpu_time_other": 101, + } + ], + "time_unit": "ns", + "utest": {}, }, { - 'name': 'BM_10PercentFaster', - 'label': '', - 'measurements': [{'time': -0.1000, 'cpu': -0.1000, - 'real_time': 100, 'real_time_other': 90, - 'cpu_time': 100, 'cpu_time_other': 90}], - 'time_unit': 'ns', - 'utest': {} + "name": "BM_10PercentFaster", + "label": "", + "measurements": [ + { + "time": -0.1000, + "cpu": -0.1000, + "real_time": 100, + "real_time_other": 90, + "cpu_time": 100, + "cpu_time_other": 90, + } + ], + "time_unit": "ns", + "utest": {}, }, { - 'name': 'BM_10PercentSlower', - 'label': '', - 'measurements': [{'time': 0.1000, 'cpu': 0.1000, - 'real_time': 100, 'real_time_other': 110, - 'cpu_time': 100, 'cpu_time_other': 110}], - 'time_unit': 'ns', - 'utest': {} + "name": "BM_10PercentSlower", + "label": "", + "measurements": [ + { + "time": 0.1000, + "cpu": 0.1000, + "real_time": 100, + "real_time_other": 110, + "cpu_time": 100, + "cpu_time_other": 110, + } + ], + "time_unit": "ns", + "utest": {}, }, { - 'name': 'BM_100xSlower', - 'label': '', - 'measurements': [{'time': 99.0000, 'cpu': 99.0000, - 'real_time': 100, 'real_time_other': 10000, - 'cpu_time': 100, 'cpu_time_other': 10000}], - 'time_unit': 'ns', - 'utest': {} + "name": "BM_100xSlower", + "label": "", + "measurements": [ + { + "time": 99.0000, + "cpu": 99.0000, + "real_time": 100, + "real_time_other": 10000, + "cpu_time": 100, + "cpu_time_other": 10000, + } + ], + "time_unit": "ns", + "utest": {}, }, { - 'name': 'BM_100xFaster', - 'label': '', - 'measurements': [{'time': -0.9900, 'cpu': -0.9900, - 'real_time': 10000, 'real_time_other': 100, - 'cpu_time': 10000, 'cpu_time_other': 100}], - 'time_unit': 'ns', - 'utest': {} + "name": "BM_100xFaster", + "label": "", + "measurements": [ + { + "time": -0.9900, + "cpu": -0.9900, + "real_time": 10000, + "real_time_other": 100, + "cpu_time": 10000, + "cpu_time_other": 100, + } + ], + "time_unit": "ns", + "utest": {}, }, { - 'name': 'BM_10PercentCPUToTime', - 'label': '', - 'measurements': [{'time': 0.1000, 'cpu': -0.1000, - 'real_time': 100, 'real_time_other': 110, - 'cpu_time': 100, 'cpu_time_other': 90}], - 'time_unit': 'ns', - 'utest': {} + "name": "BM_10PercentCPUToTime", + "label": "", + "measurements": [ + { + "time": 0.1000, + "cpu": -0.1000, + "real_time": 100, + "real_time_other": 110, + "cpu_time": 100, + "cpu_time_other": 90, + } + ], + "time_unit": "ns", + "utest": {}, }, { - 'name': 'BM_ThirdFaster', - 'label': '', - 'measurements': [{'time': -0.3333, 'cpu': -0.3334, - 'real_time': 100, 'real_time_other': 67, - 'cpu_time': 100, 'cpu_time_other': 67}], - 'time_unit': 'ns', - 'utest': {} + "name": "BM_ThirdFaster", + "label": "", + "measurements": [ + { + "time": -0.3333, + "cpu": -0.3334, + "real_time": 100, + "real_time_other": 67, + "cpu_time": 100, + "cpu_time_other": 67, + } + ], + "time_unit": "ns", + "utest": {}, }, { - 'name': 'BM_NotBadTimeUnit', - 'label': '', - 'measurements': [{'time': -0.9000, 'cpu': 0.2000, - 'real_time': 0.4, 'real_time_other': 0.04, - 'cpu_time': 0.5, 'cpu_time_other': 0.6}], - 'time_unit': 's', - 'utest': {} + "name": "BM_NotBadTimeUnit", + "label": "", + "measurements": [ + { + "time": -0.9000, + "cpu": 0.2000, + "real_time": 0.4, + "real_time_other": 0.04, + "cpu_time": 0.5, + "cpu_time_other": 0.6, + } + ], + "time_unit": "s", + "utest": {}, }, { - 'name': 'BM_hasLabel', - 'label': 'a label', - 'measurements': [{'time': 0.0000, 'cpu': 0.0000, - 'real_time': 1, 'real_time_other': 1, - 'cpu_time': 1, 'cpu_time_other': 1}], - 'time_unit': 's', - 'utest': {} + "name": "BM_hasLabel", + "label": "a label", + "measurements": [ + { + "time": 0.0000, + "cpu": 0.0000, + "real_time": 1, + "real_time_other": 1, + "cpu_time": 1, + "cpu_time_other": 1, + } + ], + "time_unit": "s", + "utest": {}, }, { - 'name': 'OVERALL_GEOMEAN', - 'label': '', - 'measurements': [{'real_time': 3.1622776601683826e-06, 'cpu_time': 3.2130844755623912e-06, - 'real_time_other': 1.9768988699420897e-07, 'cpu_time_other': 2.397447755209533e-07, - 'time': -0.8117033010153573, 'cpu': -0.7783324768278522}], - 'time_unit': 's', - 'run_type': 'aggregate', - 'aggregate_name': 'geomean', 'utest': {} + "name": "OVERALL_GEOMEAN", + "label": "", + "measurements": [ + { + "real_time": 3.1622776601683826e-06, + "cpu_time": 3.2130844755623912e-06, + "real_time_other": 1.9768988699420897e-07, + "cpu_time_other": 2.397447755209533e-07, + "time": -0.8112976497120911, + "cpu": -0.7778551721181174, + } + ], + "time_unit": "s", + "run_type": "aggregate", + "aggregate_name": "geomean", + "utest": {}, }, ] self.assertEqual(len(self.json_diff_report), len(expected_output)) for out, expected in zip( - self.json_diff_report, expected_output): - self.assertEqual(out['name'], expected['name']) - self.assertEqual(out['label'], expected['label']) - self.assertEqual(out['time_unit'], expected['time_unit']) + self.json_diff_report, expected_output, strict=True + ): + self.assertEqual(out["name"], expected["name"]) + self.assertEqual(out["label"], expected["label"]) + self.assertEqual(out["time_unit"], expected["time_unit"]) assert_utest(self, out, expected) assert_measurements(self, out, expected) @@ -612,12 +836,12 @@ class TestReportDifferenceBetweenFamilies(unittest.TestCase): def setUpClass(cls): def load_result(): import json + testInputs = os.path.join( - os.path.dirname( - os.path.realpath(__file__)), - 'Inputs') - testOutput = os.path.join(testInputs, 'test2_run.json') - with open(testOutput, 'r') as f: + os.path.dirname(os.path.realpath(__file__)), "Inputs" + ) + testOutput = os.path.join(testInputs, "test2_run.json") + with open(testOutput) as f: json = json.load(f) return json @@ -628,65 +852,110 @@ def load_result(): def test_json_diff_report_pretty_printing(self): expect_lines = [ - ['.', '-0.5000', '-0.5000', '10', '5', '10', '5'], - ['./4', '-0.5000', '-0.5000', '40', '20', '40', '20'], - ['Prefix/.', '-0.5000', '-0.5000', '20', '10', '20', '10'], - ['Prefix/./3', '-0.5000', '-0.5000', '30', '15', '30', '15'], - ['OVERALL_GEOMEAN', '-0.5000', '-0.5000', '0', '0', '0', '0'] + [".", "-0.5000", "-0.5000", "10", "5", "10", "5"], + ["./4", "-0.5000", "-0.5000", "40", "20", "40", "20"], + ["Prefix/.", "-0.5000", "-0.5000", "20", "10", "20", "10"], + ["Prefix/./3", "-0.5000", "-0.5000", "30", "15", "30", "15"], + ["OVERALL_GEOMEAN", "-0.5000", "-0.5000", "0", "0", "0", "0"], ] output_lines_with_header = print_difference_report( - self.json_diff_report, use_color=False) + self.json_diff_report, use_color=False + ) output_lines = output_lines_with_header[2:] print("\n") print("\n".join(output_lines_with_header)) self.assertEqual(len(output_lines), len(expect_lines)) - for i in range(0, len(output_lines)): - parts = [x for x in output_lines[i].split(' ') if x] + for i, output_line in enumerate(output_lines): + parts = [x for x in output_line.split(" ") if x] self.assertEqual(len(parts), 7) self.assertEqual(expect_lines[i], parts) def test_json_diff_report(self): expected_output = [ { - 'name': u'.', - 'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 10, 'real_time_other': 5, 'cpu_time': 10, 'cpu_time_other': 5}], - 'time_unit': 'ns', - 'utest': {} + "name": ".", + "measurements": [ + { + "time": -0.5, + "cpu": -0.5, + "real_time": 10, + "real_time_other": 5, + "cpu_time": 10, + "cpu_time_other": 5, + } + ], + "time_unit": "ns", + "utest": {}, }, { - 'name': u'./4', - 'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 40, 'real_time_other': 20, 'cpu_time': 40, 'cpu_time_other': 20}], - 'time_unit': 'ns', - 'utest': {}, + "name": "./4", + "measurements": [ + { + "time": -0.5, + "cpu": -0.5, + "real_time": 40, + "real_time_other": 20, + "cpu_time": 40, + "cpu_time_other": 20, + } + ], + "time_unit": "ns", + "utest": {}, }, { - 'name': u'Prefix/.', - 'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 20, 'real_time_other': 10, 'cpu_time': 20, 'cpu_time_other': 10}], - 'time_unit': 'ns', - 'utest': {} + "name": "Prefix/.", + "measurements": [ + { + "time": -0.5, + "cpu": -0.5, + "real_time": 20, + "real_time_other": 10, + "cpu_time": 20, + "cpu_time_other": 10, + } + ], + "time_unit": "ns", + "utest": {}, }, { - 'name': u'Prefix/./3', - 'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 30, 'real_time_other': 15, 'cpu_time': 30, 'cpu_time_other': 15}], - 'time_unit': 'ns', - 'utest': {} + "name": "Prefix/./3", + "measurements": [ + { + "time": -0.5, + "cpu": -0.5, + "real_time": 30, + "real_time_other": 15, + "cpu_time": 30, + "cpu_time_other": 15, + } + ], + "time_unit": "ns", + "utest": {}, }, { - 'name': 'OVERALL_GEOMEAN', - 'measurements': [{'real_time': 2.213363839400641e-08, 'cpu_time': 2.213363839400641e-08, - 'real_time_other': 1.1066819197003185e-08, 'cpu_time_other': 1.1066819197003185e-08, - 'time': -0.5000000000000009, 'cpu': -0.5000000000000009}], - 'time_unit': 's', - 'run_type': 'aggregate', - 'aggregate_name': 'geomean', - 'utest': {} - } + "name": "OVERALL_GEOMEAN", + "measurements": [ + { + "real_time": 2.213363839400641e-08, + "cpu_time": 2.213363839400641e-08, + "real_time_other": 1.1066819197003185e-08, + "cpu_time_other": 1.1066819197003185e-08, + "time": -0.5000000000000009, + "cpu": -0.5000000000000009, + } + ], + "time_unit": "s", + "run_type": "aggregate", + "aggregate_name": "geomean", + "utest": {}, + }, ] self.assertEqual(len(self.json_diff_report), len(expected_output)) for out, expected in zip( - self.json_diff_report, expected_output): - self.assertEqual(out['name'], expected['name']) - self.assertEqual(out['time_unit'], expected['time_unit']) + self.json_diff_report, expected_output, strict=True + ): + self.assertEqual(out["name"], expected["name"]) + self.assertEqual(out["time_unit"], expected["time_unit"]) assert_utest(self, out, expected) assert_measurements(self, out, expected) @@ -696,424 +965,495 @@ class TestReportDifferenceWithUTest(unittest.TestCase): def setUpClass(cls): def load_results(): import json + testInputs = os.path.join( - os.path.dirname( - os.path.realpath(__file__)), - 'Inputs') - testOutput1 = os.path.join(testInputs, 'test3_run0.json') - testOutput2 = os.path.join(testInputs, 'test3_run1.json') - with open(testOutput1, 'r') as f: + os.path.dirname(os.path.realpath(__file__)), "Inputs" + ) + testOutput1 = os.path.join(testInputs, "test3_run0.json") + testOutput2 = os.path.join(testInputs, "test3_run1.json") + with open(testOutput1) as f: json1 = json.load(f) - with open(testOutput2, 'r') as f: + with open(testOutput2) as f: json2 = json.load(f) return json1, json2 json1, json2 = load_results() - cls.json_diff_report = get_difference_report( - json1, json2, utest=True) + cls.json_diff_report = get_difference_report(json1, json2, utest=True) def test_json_diff_report_pretty_printing(self): expect_lines = [ - ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'], - ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'], - ['BM_Two', '-0.1250', '-0.1628', '8', '7', '86', '72'], - ['BM_Two_pvalue', - '1.0000', - '0.6667', - 'U', - 'Test,', - 'Repetitions:', - '2', - 'vs', - '2.', - 'WARNING:', - 'Results', - 'unreliable!', - '9+', - 'repetitions', - 'recommended.'], - ['short', '-0.1250', '-0.0625', '8', '7', '80', '75'], - ['short', '-0.4325', '-0.1351', '8', '5', '77', '67'], - ['short_pvalue', - '0.7671', - '0.2000', - 'U', - 'Test,', - 'Repetitions:', - '2', - 'vs', - '3.', - 'WARNING:', - 'Results', - 'unreliable!', - '9+', - 'repetitions', - 'recommended.'], - ['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'], - ['OVERALL_GEOMEAN', '+1.6405', '-0.6985', '0', '0', '0', '0'] + ["BM_One", "-0.1000", "+0.1000", "10", "9", "100", "110"], + ["BM_Two", "+0.1111", "-0.0111", "9", "10", "90", "89"], + ["BM_Two", "-0.1250", "-0.1628", "8", "7", "86", "72"], + [ + "BM_Two_pvalue", + "1.0000", + "0.6667", + "U", + "Test,", + "Repetitions:", + "2", + "vs", + "2.", + "WARNING:", + "Results", + "unreliable!", + "9+", + "repetitions", + "recommended.", + ], + ["short", "-0.1250", "-0.0625", "8", "7", "80", "75"], + ["short", "-0.4325", "-0.1351", "8", "5", "77", "67"], + [ + "short_pvalue", + "0.7671", + "0.2000", + "U", + "Test,", + "Repetitions:", + "2", + "vs", + "3.", + "WARNING:", + "Results", + "unreliable!", + "9+", + "repetitions", + "recommended.", + ], + ["medium", "-0.3750", "-0.3375", "8", "5", "80", "53"], + ["OVERALL_GEOMEAN", "+1.6405", "-0.6985", "0", "0", "0", "0"], ] output_lines_with_header = print_difference_report( - self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False) + self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False + ) output_lines = output_lines_with_header[2:] print("\n") print("\n".join(output_lines_with_header)) self.assertEqual(len(output_lines), len(expect_lines)) - for i in range(0, len(output_lines)): - parts = [x for x in output_lines[i].split(' ') if x] + for i, output_line in enumerate(output_lines): + parts = [x for x in output_line.split(" ") if x] self.assertEqual(expect_lines[i], parts) def test_json_diff_report_pretty_printing_aggregates_only(self): expect_lines = [ - ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'], - ['BM_Two_pvalue', - '1.0000', - '0.6667', - 'U', - 'Test,', - 'Repetitions:', - '2', - 'vs', - '2.', - 'WARNING:', - 'Results', - 'unreliable!', - '9+', - 'repetitions', - 'recommended.'], - ['short', '-0.1250', '-0.0625', '8', '7', '80', '75'], - ['short', '-0.4325', '-0.1351', '8', '5', '77', '67'], - ['short_pvalue', - '0.7671', - '0.2000', - 'U', - 'Test,', - 'Repetitions:', - '2', - 'vs', - '3.', - 'WARNING:', - 'Results', - 'unreliable!', - '9+', - 'repetitions', - 'recommended.'], - ['OVERALL_GEOMEAN', '+1.6405', '-0.6985', '0', '0', '0', '0'] + ["BM_One", "-0.1000", "+0.1000", "10", "9", "100", "110"], + [ + "BM_Two_pvalue", + "1.0000", + "0.6667", + "U", + "Test,", + "Repetitions:", + "2", + "vs", + "2.", + "WARNING:", + "Results", + "unreliable!", + "9+", + "repetitions", + "recommended.", + ], + ["short", "-0.1250", "-0.0625", "8", "7", "80", "75"], + ["short", "-0.4325", "-0.1351", "8", "5", "77", "67"], + [ + "short_pvalue", + "0.7671", + "0.2000", + "U", + "Test,", + "Repetitions:", + "2", + "vs", + "3.", + "WARNING:", + "Results", + "unreliable!", + "9+", + "repetitions", + "recommended.", + ], + ["OVERALL_GEOMEAN", "+1.6405", "-0.6985", "0", "0", "0", "0"], ] output_lines_with_header = print_difference_report( - self.json_diff_report, include_aggregates_only=True, utest=True, utest_alpha=0.05, use_color=False) + self.json_diff_report, + include_aggregates_only=True, + utest=True, + utest_alpha=0.05, + use_color=False, + ) output_lines = output_lines_with_header[2:] print("\n") print("\n".join(output_lines_with_header)) self.assertEqual(len(output_lines), len(expect_lines)) - for i in range(0, len(output_lines)): - parts = [x for x in output_lines[i].split(' ') if x] + for i, output_line in enumerate(output_lines): + parts = [x for x in output_line.split(" ") if x] self.assertEqual(expect_lines[i], parts) def test_json_diff_report(self): expected_output = [ { - 'name': u'BM_One', - 'measurements': [ - {'time': -0.1, - 'cpu': 0.1, - 'real_time': 10, - 'real_time_other': 9, - 'cpu_time': 100, - 'cpu_time_other': 110} + "name": "BM_One", + "measurements": [ + { + "time": -0.1, + "cpu": 0.1, + "real_time": 10, + "real_time_other": 9, + "cpu_time": 100, + "cpu_time_other": 110, + } ], - 'time_unit': 'ns', - 'utest': {} + "time_unit": "ns", + "utest": {}, }, { - 'name': u'BM_Two', - 'measurements': [ - {'time': 0.1111111111111111, - 'cpu': -0.011111111111111112, - 'real_time': 9, - 'real_time_other': 10, - 'cpu_time': 90, - 'cpu_time_other': 89}, - {'time': -0.125, 'cpu': -0.16279069767441862, 'real_time': 8, - 'real_time_other': 7, 'cpu_time': 86, 'cpu_time_other': 72} + "name": "BM_Two", + "measurements": [ + { + "time": 0.1111111111111111, + "cpu": -0.011111111111111112, + "real_time": 9, + "real_time_other": 10, + "cpu_time": 90, + "cpu_time_other": 89, + }, + { + "time": -0.125, + "cpu": -0.16279069767441862, + "real_time": 8, + "real_time_other": 7, + "cpu_time": 86, + "cpu_time_other": 72, + }, ], - 'time_unit': 'ns', - 'utest': { - 'have_optimal_repetitions': False, 'cpu_pvalue': 0.6666666666666666, 'time_pvalue': 1.0 - } + "time_unit": "ns", + "utest": { + "have_optimal_repetitions": False, + "cpu_pvalue": 0.6666666666666666, + "time_pvalue": 1.0, + }, }, { - 'name': u'short', - 'measurements': [ - {'time': -0.125, - 'cpu': -0.0625, - 'real_time': 8, - 'real_time_other': 7, - 'cpu_time': 80, - 'cpu_time_other': 75}, - {'time': -0.4325, - 'cpu': -0.13506493506493514, - 'real_time': 8, - 'real_time_other': 4.54, - 'cpu_time': 77, - 'cpu_time_other': 66.6} + "name": "short", + "measurements": [ + { + "time": -0.125, + "cpu": -0.0625, + "real_time": 8, + "real_time_other": 7, + "cpu_time": 80, + "cpu_time_other": 75, + }, + { + "time": -0.4325, + "cpu": -0.13506493506493514, + "real_time": 8, + "real_time_other": 4.54, + "cpu_time": 77, + "cpu_time_other": 66.6, + }, ], - 'time_unit': 'ns', - 'utest': { - 'have_optimal_repetitions': False, 'cpu_pvalue': 0.2, 'time_pvalue': 0.7670968684102772 - } + "time_unit": "ns", + "utest": { + "have_optimal_repetitions": False, + "cpu_pvalue": 0.2, + "time_pvalue": 0.7670968684102772, + }, }, { - 'name': u'medium', - 'measurements': [ - {'time': -0.375, - 'cpu': -0.3375, - 'real_time': 8, - 'real_time_other': 5, - 'cpu_time': 80, - 'cpu_time_other': 53} + "name": "medium", + "measurements": [ + { + "time": -0.375, + "cpu": -0.3375, + "real_time": 8, + "real_time_other": 5, + "cpu_time": 80, + "cpu_time_other": 53, + } ], - 'time_unit': 'ns', - 'utest': {} + "time_unit": "ns", + "utest": {}, }, { - 'name': 'OVERALL_GEOMEAN', - 'measurements': [{'real_time': 8.48528137423858e-09, 'cpu_time': 8.441336246629233e-08, - 'real_time_other': 2.2405267593145244e-08, 'cpu_time_other': 2.5453661413660466e-08, - 'time': 1.6404861082353634, 'cpu': -0.6984640740519662}], - 'time_unit': 's', - 'run_type': 'aggregate', - 'aggregate_name': 'geomean', - 'utest': {} - } + "name": "OVERALL_GEOMEAN", + "measurements": [ + { + "real_time": 8.48528137423858e-09, + "cpu_time": 8.441336246629233e-08, + "real_time_other": 2.2405267593145244e-08, + "cpu_time_other": 2.5453661413660466e-08, + "time": 1.6404861082353634, + "cpu": -0.6984640740519662, + } + ], + "time_unit": "s", + "run_type": "aggregate", + "aggregate_name": "geomean", + "utest": {}, + }, ] self.assertEqual(len(self.json_diff_report), len(expected_output)) for out, expected in zip( - self.json_diff_report, expected_output): - self.assertEqual(out['name'], expected['name']) - self.assertEqual(out['time_unit'], expected['time_unit']) + self.json_diff_report, expected_output, strict=True + ): + self.assertEqual(out["name"], expected["name"]) + self.assertEqual(out["time_unit"], expected["time_unit"]) assert_utest(self, out, expected) assert_measurements(self, out, expected) class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly( - unittest.TestCase): + unittest.TestCase +): @classmethod def setUpClass(cls): def load_results(): import json + testInputs = os.path.join( - os.path.dirname( - os.path.realpath(__file__)), - 'Inputs') - testOutput1 = os.path.join(testInputs, 'test3_run0.json') - testOutput2 = os.path.join(testInputs, 'test3_run1.json') - with open(testOutput1, 'r') as f: + os.path.dirname(os.path.realpath(__file__)), "Inputs" + ) + testOutput1 = os.path.join(testInputs, "test3_run0.json") + testOutput2 = os.path.join(testInputs, "test3_run1.json") + with open(testOutput1) as f: json1 = json.load(f) - with open(testOutput2, 'r') as f: + with open(testOutput2) as f: json2 = json.load(f) return json1, json2 json1, json2 = load_results() - cls.json_diff_report = get_difference_report( - json1, json2, utest=True) + cls.json_diff_report = get_difference_report(json1, json2, utest=True) def test_json_diff_report_pretty_printing(self): expect_lines = [ - ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'], - ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'], - ['BM_Two', '-0.1250', '-0.1628', '8', '7', '86', '72'], - ['BM_Two_pvalue', - '1.0000', - '0.6667', - 'U', - 'Test,', - 'Repetitions:', - '2', - 'vs', - '2.', - 'WARNING:', - 'Results', - 'unreliable!', - '9+', - 'repetitions', - 'recommended.'], - ['short', '-0.1250', '-0.0625', '8', '7', '80', '75'], - ['short', '-0.4325', '-0.1351', '8', '5', '77', '67'], - ['short_pvalue', - '0.7671', - '0.2000', - 'U', - 'Test,', - 'Repetitions:', - '2', - 'vs', - '3.', - 'WARNING:', - 'Results', - 'unreliable!', - '9+', - 'repetitions', - 'recommended.'], - ['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'], - ['OVERALL_GEOMEAN', '+1.6405', '-0.6985', '0', '0', '0', '0'] + ["BM_One", "-0.1000", "+0.1000", "10", "9", "100", "110"], + ["BM_Two", "+0.1111", "-0.0111", "9", "10", "90", "89"], + ["BM_Two", "-0.1250", "-0.1628", "8", "7", "86", "72"], + [ + "BM_Two_pvalue", + "1.0000", + "0.6667", + "U", + "Test,", + "Repetitions:", + "2", + "vs", + "2.", + "WARNING:", + "Results", + "unreliable!", + "9+", + "repetitions", + "recommended.", + ], + ["short", "-0.1250", "-0.0625", "8", "7", "80", "75"], + ["short", "-0.4325", "-0.1351", "8", "5", "77", "67"], + [ + "short_pvalue", + "0.7671", + "0.2000", + "U", + "Test,", + "Repetitions:", + "2", + "vs", + "3.", + "WARNING:", + "Results", + "unreliable!", + "9+", + "repetitions", + "recommended.", + ], + ["medium", "-0.3750", "-0.3375", "8", "5", "80", "53"], + ["OVERALL_GEOMEAN", "+1.6405", "-0.6985", "0", "0", "0", "0"], ] output_lines_with_header = print_difference_report( - self.json_diff_report, - utest=True, utest_alpha=0.05, use_color=False) + self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False + ) output_lines = output_lines_with_header[2:] print("\n") print("\n".join(output_lines_with_header)) self.assertEqual(len(output_lines), len(expect_lines)) - for i in range(0, len(output_lines)): - parts = [x for x in output_lines[i].split(' ') if x] + for i, output_line in enumerate(output_lines): + parts = [x for x in output_line.split(" ") if x] self.assertEqual(expect_lines[i], parts) def test_json_diff_report(self): expected_output = [ { - 'name': u'BM_One', - 'measurements': [ - {'time': -0.1, - 'cpu': 0.1, - 'real_time': 10, - 'real_time_other': 9, - 'cpu_time': 100, - 'cpu_time_other': 110} + "name": "BM_One", + "measurements": [ + { + "time": -0.1, + "cpu": 0.1, + "real_time": 10, + "real_time_other": 9, + "cpu_time": 100, + "cpu_time_other": 110, + } ], - 'time_unit': 'ns', - 'utest': {} + "time_unit": "ns", + "utest": {}, }, { - 'name': u'BM_Two', - 'measurements': [ - {'time': 0.1111111111111111, - 'cpu': -0.011111111111111112, - 'real_time': 9, - 'real_time_other': 10, - 'cpu_time': 90, - 'cpu_time_other': 89}, - {'time': -0.125, 'cpu': -0.16279069767441862, 'real_time': 8, - 'real_time_other': 7, 'cpu_time': 86, 'cpu_time_other': 72} + "name": "BM_Two", + "measurements": [ + { + "time": 0.1111111111111111, + "cpu": -0.011111111111111112, + "real_time": 9, + "real_time_other": 10, + "cpu_time": 90, + "cpu_time_other": 89, + }, + { + "time": -0.125, + "cpu": -0.16279069767441862, + "real_time": 8, + "real_time_other": 7, + "cpu_time": 86, + "cpu_time_other": 72, + }, ], - 'time_unit': 'ns', - 'utest': { - 'have_optimal_repetitions': False, 'cpu_pvalue': 0.6666666666666666, 'time_pvalue': 1.0 - } + "time_unit": "ns", + "utest": { + "have_optimal_repetitions": False, + "cpu_pvalue": 0.6666666666666666, + "time_pvalue": 1.0, + }, }, { - 'name': u'short', - 'measurements': [ - {'time': -0.125, - 'cpu': -0.0625, - 'real_time': 8, - 'real_time_other': 7, - 'cpu_time': 80, - 'cpu_time_other': 75}, - {'time': -0.4325, - 'cpu': -0.13506493506493514, - 'real_time': 8, - 'real_time_other': 4.54, - 'cpu_time': 77, - 'cpu_time_other': 66.6} + "name": "short", + "measurements": [ + { + "time": -0.125, + "cpu": -0.0625, + "real_time": 8, + "real_time_other": 7, + "cpu_time": 80, + "cpu_time_other": 75, + }, + { + "time": -0.4325, + "cpu": -0.13506493506493514, + "real_time": 8, + "real_time_other": 4.54, + "cpu_time": 77, + "cpu_time_other": 66.6, + }, ], - 'time_unit': 'ns', - 'utest': { - 'have_optimal_repetitions': False, 'cpu_pvalue': 0.2, 'time_pvalue': 0.7670968684102772 - } + "time_unit": "ns", + "utest": { + "have_optimal_repetitions": False, + "cpu_pvalue": 0.2, + "time_pvalue": 0.7670968684102772, + }, }, { - 'name': u'medium', - 'measurements': [ - {'real_time_other': 5, - 'cpu_time': 80, - 'time': -0.375, - 'real_time': 8, - 'cpu_time_other': 53, - 'cpu': -0.3375 - } + "name": "medium", + "measurements": [ + { + "real_time_other": 5, + "cpu_time": 80, + "time": -0.375, + "real_time": 8, + "cpu_time_other": 53, + "cpu": -0.3375, + } ], - 'utest': {}, - 'time_unit': u'ns', - 'aggregate_name': '' + "utest": {}, + "time_unit": "ns", + "aggregate_name": "", }, { - 'name': 'OVERALL_GEOMEAN', - 'measurements': [{'real_time': 8.48528137423858e-09, 'cpu_time': 8.441336246629233e-08, - 'real_time_other': 2.2405267593145244e-08, 'cpu_time_other': 2.5453661413660466e-08, - 'time': 1.6404861082353634, 'cpu': -0.6984640740519662}], - 'time_unit': 's', - 'run_type': 'aggregate', - 'aggregate_name': 'geomean', - 'utest': {} - } + "name": "OVERALL_GEOMEAN", + "measurements": [ + { + "real_time": 8.48528137423858e-09, + "cpu_time": 8.441336246629233e-08, + "real_time_other": 2.2405267593145244e-08, + "cpu_time_other": 2.5453661413660466e-08, + "time": 1.6404861082353634, + "cpu": -0.6984640740519662, + } + ], + "time_unit": "s", + "run_type": "aggregate", + "aggregate_name": "geomean", + "utest": {}, + }, ] self.assertEqual(len(self.json_diff_report), len(expected_output)) for out, expected in zip( - self.json_diff_report, expected_output): - self.assertEqual(out['name'], expected['name']) - self.assertEqual(out['time_unit'], expected['time_unit']) + self.json_diff_report, expected_output, strict=True + ): + self.assertEqual(out["name"], expected["name"]) + self.assertEqual(out["time_unit"], expected["time_unit"]) assert_utest(self, out, expected) assert_measurements(self, out, expected) -class TestReportDifferenceForPercentageAggregates( - unittest.TestCase): +class TestReportDifferenceForPercentageAggregates(unittest.TestCase): @classmethod def setUpClass(cls): def load_results(): import json + testInputs = os.path.join( - os.path.dirname( - os.path.realpath(__file__)), - 'Inputs') - testOutput1 = os.path.join(testInputs, 'test4_run0.json') - testOutput2 = os.path.join(testInputs, 'test4_run1.json') - with open(testOutput1, 'r') as f: + os.path.dirname(os.path.realpath(__file__)), "Inputs" + ) + testOutput1 = os.path.join(testInputs, "test4_run0.json") + testOutput2 = os.path.join(testInputs, "test4_run1.json") + with open(testOutput1) as f: json1 = json.load(f) - with open(testOutput2, 'r') as f: + with open(testOutput2) as f: json2 = json.load(f) return json1, json2 json1, json2 = load_results() - cls.json_diff_report = get_difference_report( - json1, json2, utest=True) + cls.json_diff_report = get_difference_report(json1, json2, utest=True) def test_json_diff_report_pretty_printing(self): - expect_lines = [ - ['whocares', '-0.5000', '+0.5000', '0', '0', '0', '0'] - ] + expect_lines = [["whocares", "-0.5000", "+0.5000", "0", "0", "0", "0"]] output_lines_with_header = print_difference_report( - self.json_diff_report, - utest=True, utest_alpha=0.05, use_color=False) + self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False + ) output_lines = output_lines_with_header[2:] print("\n") print("\n".join(output_lines_with_header)) self.assertEqual(len(output_lines), len(expect_lines)) - for i in range(0, len(output_lines)): - parts = [x for x in output_lines[i].split(' ') if x] + for i, output_line in enumerate(output_lines): + parts = [x for x in output_line.split(" ") if x] self.assertEqual(expect_lines[i], parts) def test_json_diff_report(self): expected_output = [ { - 'name': u'whocares', - 'measurements': [ - {'time': -0.5, - 'cpu': 0.5, - 'real_time': 0.01, - 'real_time_other': 0.005, - 'cpu_time': 0.10, - 'cpu_time_other': 0.15} + "name": "whocares", + "measurements": [ + { + "time": -0.5, + "cpu": 0.5, + "real_time": 0.01, + "real_time_other": 0.005, + "cpu_time": 0.10, + "cpu_time_other": 0.15, + } ], - 'time_unit': 'ns', - 'utest': {} + "time_unit": "ns", + "utest": {}, } ] self.assertEqual(len(self.json_diff_report), len(expected_output)) for out, expected in zip( - self.json_diff_report, expected_output): - self.assertEqual(out['name'], expected['name']) - self.assertEqual(out['time_unit'], expected['time_unit']) + self.json_diff_report, expected_output, strict=True + ): + self.assertEqual(out["name"], expected["name"]) + self.assertEqual(out["time_unit"], expected["time_unit"]) assert_utest(self, out, expected) assert_measurements(self, out, expected) @@ -1123,12 +1463,12 @@ class TestReportSorting(unittest.TestCase): def setUpClass(cls): def load_result(): import json + testInputs = os.path.join( - os.path.dirname( - os.path.realpath(__file__)), - 'Inputs') - testOutput = os.path.join(testInputs, 'test4_run.json') - with open(testOutput, 'r') as f: + os.path.dirname(os.path.realpath(__file__)), "Inputs" + ) + testOutput = os.path.join(testInputs, "test4_run.json") + with open(testOutput) as f: json = json.load(f) return json @@ -1149,45 +1489,145 @@ def test_json_diff_report_pretty_printing(self): "91 family 1 instance 0 aggregate", "90 family 1 instance 1 repetition 0", "89 family 1 instance 1 repetition 1", - "88 family 1 instance 1 aggregate" + "88 family 1 instance 1 aggregate", ] - for n in range(len(self.json['benchmarks']) ** 2): - random.shuffle(self.json['benchmarks']) + for _n in range(len(self.json["benchmarks"]) ** 2): + random.shuffle(self.json["benchmarks"]) sorted_benchmarks = util.sort_benchmark_results(self.json)[ - 'benchmarks'] + "benchmarks" + ] self.assertEqual(len(expected_names), len(sorted_benchmarks)) - for out, expected in zip(sorted_benchmarks, expected_names): - self.assertEqual(out['name'], expected) + for out, expected in zip( + sorted_benchmarks, expected_names, strict=True + ): + self.assertEqual(out["name"], expected) + + +class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly2( + unittest.TestCase +): + @classmethod + def setUpClass(cls): + def load_results(): + import json + + testInputs = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "Inputs" + ) + testOutput1 = os.path.join(testInputs, "test5_run0.json") + testOutput2 = os.path.join(testInputs, "test5_run1.json") + with open(testOutput1) as f: + json1 = json.load(f) + json1["benchmarks"] = [ + json1["benchmarks"][0] for i in range(1000) + ] + with open(testOutput2) as f: + json2 = json.load(f) + json2["benchmarks"] = [ + json2["benchmarks"][0] for i in range(1000) + ] + return json1, json2 + + json1, json2 = load_results() + cls.json_diff_report = get_difference_report(json1, json2, utest=True) + + def test_json_diff_report_pretty_printing(self): + expect_line = [ + "BM_ManyRepetitions_pvalue", + "0.0000", + "0.0000", + "U", + "Test,", + "Repetitions:", + "1000", + "vs", + "1000", + ] + output_lines_with_header = print_difference_report( + self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False + ) + output_lines = output_lines_with_header[2:] + found = False + for output_line in output_lines: + parts = [x for x in output_line.split(" ") if x] + found = expect_line == parts + if found: + break + self.assertTrue(found) + + def test_json_diff_report(self): + expected_output = [ + { + "name": "BM_ManyRepetitions", + "label": "", + "time_unit": "s", + "run_type": "", + "aggregate_name": "", + "utest": { + "have_optimal_repetitions": True, + "cpu_pvalue": 0.0, + "time_pvalue": 0.0, + "nr_of_repetitions": 1000, + "nr_of_repetitions_other": 1000, + }, + }, + { + "name": "OVERALL_GEOMEAN", + "label": "", + "measurements": [ + { + "real_time": 1.0, + "cpu_time": 1000.000000000069, + "real_time_other": 1000.000000000069, + "cpu_time_other": 1.0, + "time": 999.000000000069, + "cpu": -0.9990000000000001, + } + ], + "time_unit": "s", + "run_type": "aggregate", + "aggregate_name": "geomean", + "utest": {}, + }, + ] + self.assertEqual(len(self.json_diff_report), len(expected_output)) + for out, expected in zip( + self.json_diff_report, expected_output, strict=True + ): + self.assertEqual(out["name"], expected["name"]) + self.assertEqual(out["time_unit"], expected["time_unit"]) + assert_utest(self, out, expected) def assert_utest(unittest_instance, lhs, rhs): - if lhs['utest']: + if lhs["utest"]: unittest_instance.assertAlmostEqual( - lhs['utest']['cpu_pvalue'], - rhs['utest']['cpu_pvalue']) + lhs["utest"]["cpu_pvalue"], rhs["utest"]["cpu_pvalue"] + ) unittest_instance.assertAlmostEqual( - lhs['utest']['time_pvalue'], - rhs['utest']['time_pvalue']) + lhs["utest"]["time_pvalue"], rhs["utest"]["time_pvalue"] + ) unittest_instance.assertEqual( - lhs['utest']['have_optimal_repetitions'], - rhs['utest']['have_optimal_repetitions']) + lhs["utest"]["have_optimal_repetitions"], + rhs["utest"]["have_optimal_repetitions"], + ) else: # lhs is empty. assert if rhs is not. - unittest_instance.assertEqual(lhs['utest'], rhs['utest']) + unittest_instance.assertEqual(lhs["utest"], rhs["utest"]) def assert_measurements(unittest_instance, lhs, rhs): - for m1, m2 in zip(lhs['measurements'], rhs['measurements']): - unittest_instance.assertEqual(m1['real_time'], m2['real_time']) - unittest_instance.assertEqual(m1['cpu_time'], m2['cpu_time']) + for m1, m2 in zip(lhs["measurements"], rhs["measurements"], strict=False): + unittest_instance.assertEqual(m1["real_time"], m2["real_time"]) + unittest_instance.assertEqual(m1["cpu_time"], m2["cpu_time"]) # m1['time'] and m1['cpu'] hold values which are being calculated, # and therefore we must use almost-equal pattern. - unittest_instance.assertAlmostEqual(m1['time'], m2['time'], places=4) - unittest_instance.assertAlmostEqual(m1['cpu'], m2['cpu'], places=4) + unittest_instance.assertAlmostEqual(m1["time"], m2["time"], places=4) + unittest_instance.assertAlmostEqual(m1["cpu"], m2["cpu"], places=4) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/tools/gbench/util.py b/tools/gbench/util.py index 5d0012c0cb..7847e65444 100644 --- a/tools/gbench/util.py +++ b/tools/gbench/util.py @@ -1,18 +1,20 @@ -"""util.py - General utilities for running, loading, and processing benchmarks +"""util.py - General utilities for running, loading, and processing +benchmarks """ + import json import os -import tempfile +import re import subprocess import sys -import functools +import tempfile # Input file type enumeration IT_Invalid = 0 IT_JSON = 1 IT_Executable = 2 -_num_magic_bytes = 2 if sys.platform.startswith('win') else 4 +_num_magic_bytes = 2 if sys.platform.startswith("win") else 4 def is_executable_file(filename): @@ -23,21 +25,21 @@ def is_executable_file(filename): """ if not os.path.isfile(filename): return False - with open(filename, mode='rb') as f: + with open(filename, mode="rb") as f: magic_bytes = f.read(_num_magic_bytes) - if sys.platform == 'darwin': + if sys.platform == "darwin": return magic_bytes in [ - b'\xfe\xed\xfa\xce', # MH_MAGIC - b'\xce\xfa\xed\xfe', # MH_CIGAM - b'\xfe\xed\xfa\xcf', # MH_MAGIC_64 - b'\xcf\xfa\xed\xfe', # MH_CIGAM_64 - b'\xca\xfe\xba\xbe', # FAT_MAGIC - b'\xbe\xba\xfe\xca' # FAT_CIGAM + b"\xfe\xed\xfa\xce", # MH_MAGIC + b"\xce\xfa\xed\xfe", # MH_CIGAM + b"\xfe\xed\xfa\xcf", # MH_MAGIC_64 + b"\xcf\xfa\xed\xfe", # MH_CIGAM_64 + b"\xca\xfe\xba\xbe", # FAT_MAGIC + b"\xbe\xba\xfe\xca", # FAT_CIGAM ] - elif sys.platform.startswith('win'): - return magic_bytes == b'MZ' + elif sys.platform.startswith("win"): + return magic_bytes == b"MZ" else: - return magic_bytes == b'\x7FELF' + return magic_bytes == b"\x7fELF" def is_json_file(filename): @@ -46,7 +48,7 @@ def is_json_file(filename): 'False' otherwise. """ try: - with open(filename, 'r') as f: + with open(filename) as f: json.load(f) return True except BaseException: @@ -58,7 +60,7 @@ def classify_input_file(filename): """ Return a tuple (type, msg) where 'type' specifies the classified type of 'filename'. If 'type' is 'IT_Invalid' then 'msg' is a human readable - string represeting the error. + string representing the error. """ ftype = IT_Invalid err_msg = None @@ -71,7 +73,10 @@ def classify_input_file(filename): elif is_json_file(filename): ftype = IT_JSON else: - err_msg = "'%s' does not name a valid benchmark executable or JSON file" % filename + err_msg = ( + "'%s' does not name a valid benchmark executable or JSON file" + % filename + ) return ftype, err_msg @@ -94,11 +99,12 @@ def find_benchmark_flag(prefix, benchmark_flags): if it is found return the arg it specifies. If specified more than once the last value is returned. If the flag is not found None is returned. """ - assert prefix.startswith('--') and prefix.endswith('=') + assert prefix.startswith("--") + assert prefix.endswith("=") result = None for f in benchmark_flags: if f.startswith(prefix): - result = f[len(prefix):] + result = f[len(prefix) :] return result @@ -107,33 +113,71 @@ def remove_benchmark_flags(prefix, benchmark_flags): Return a new list containing the specified benchmark_flags except those with the specified prefix. """ - assert prefix.startswith('--') and prefix.endswith('=') + assert prefix.startswith("--") + assert prefix.endswith("=") return [f for f in benchmark_flags if not f.startswith(prefix)] -def load_benchmark_results(fname): +def load_benchmark_results(fname, benchmark_filter): """ Read benchmark output from a file and return the JSON object. + + Apply benchmark_filter, a regular expression, with nearly the same + semantics of the --benchmark_filter argument. May be None. + Note: the Python regular expression engine is used instead of the + one used by the C++ code, which may produce different results + in complex cases. + REQUIRES: 'fname' names a file containing JSON benchmark output. """ - with open(fname, 'r') as f: - return json.load(f) + + def benchmark_wanted(benchmark): + if benchmark_filter is None: + return True + name = benchmark.get("run_name", None) or benchmark["name"] + return re.search(benchmark_filter, name) is not None + + with open(fname) as f: + results = json.load(f) + if "json_schema_version" in results.get("context", {}): + json_schema_version = results["context"]["json_schema_version"] + if json_schema_version != 1: + print( + f"In {fname}, got unsupported JSON schema version:" + f" {json_schema_version}, expected 1" + ) + sys.exit(1) + if "benchmarks" in results: + results["benchmarks"] = list( + filter(benchmark_wanted, results["benchmarks"]) + ) + return results def sort_benchmark_results(result): - benchmarks = result['benchmarks'] + benchmarks = result["benchmarks"] # From inner key to the outer key! benchmarks = sorted( - benchmarks, key=lambda benchmark: benchmark['repetition_index'] if 'repetition_index' in benchmark else -1) + benchmarks, + key=lambda benchmark: benchmark.get("repetition_index", -1), + ) benchmarks = sorted( - benchmarks, key=lambda benchmark: 1 if 'run_type' in benchmark and benchmark['run_type'] == "aggregate" else 0) + benchmarks, + key=lambda benchmark: 1 + if "run_type" in benchmark and benchmark["run_type"] == "aggregate" + else 0, + ) benchmarks = sorted( - benchmarks, key=lambda benchmark: benchmark['per_family_instance_index'] if 'per_family_instance_index' in benchmark else -1) + benchmarks, + key=lambda benchmark: benchmark.get("per_family_instance_index", -1), + ) benchmarks = sorted( - benchmarks, key=lambda benchmark: benchmark['family_index'] if 'family_index' in benchmark else -1) + benchmarks, + key=lambda benchmark: benchmark.get("family_index", -1), + ) - result['benchmarks'] = benchmarks + result["benchmarks"] = benchmarks return result @@ -144,23 +188,24 @@ def run_benchmark(exe_name, benchmark_flags): real time console output. RETURNS: A JSON object representing the benchmark output """ - output_name = find_benchmark_flag('--benchmark_out=', - benchmark_flags) + output_name = find_benchmark_flag("--benchmark_out=", benchmark_flags) is_temp_output = False if output_name is None: is_temp_output = True thandle, output_name = tempfile.mkstemp() os.close(thandle) - benchmark_flags = list(benchmark_flags) + \ - ['--benchmark_out=%s' % output_name] + benchmark_flags = [ + *list(benchmark_flags), + "--benchmark_out=%s" % output_name, + ] - cmd = [exe_name] + benchmark_flags - print("RUNNING: %s" % ' '.join(cmd)) + cmd = [exe_name, *benchmark_flags] + print("RUNNING: %s" % " ".join(cmd)) exitCode = subprocess.call(cmd) if exitCode != 0: - print('TEST FAILED...') + print("TEST FAILED...") sys.exit(exitCode) - json_res = load_benchmark_results(output_name) + json_res = load_benchmark_results(output_name, None) if is_temp_output: os.unlink(output_name) return json_res @@ -175,7 +220,10 @@ def run_or_load_benchmark(filename, benchmark_flags): """ ftype = check_input_file(filename) if ftype == IT_JSON: - return load_benchmark_results(filename) + benchmark_filter = find_benchmark_flag( + "--benchmark_filter=", benchmark_flags + ) + return load_benchmark_results(filename, benchmark_filter) if ftype == IT_Executable: return run_benchmark(filename, benchmark_flags) - raise ValueError('Unknown file type %s' % ftype) + raise ValueError("Unknown file type %s" % ftype) diff --git a/tools/libpfm.BUILD.bazel b/tools/libpfm.BUILD.bazel new file mode 100644 index 0000000000..30b585452d --- /dev/null +++ b/tools/libpfm.BUILD.bazel @@ -0,0 +1,242 @@ +"""Build rule for libpfm, which is required to collect performance counters for BENCHMARK_ENABLE_LIBPFM builds.""" + +load("@rules_cc//cc:defs.bzl", "cc_library") + +AARCH32_SRCS_COMMON = [ + "lib/pfmlib_arm.c", + "lib/pfmlib_arm_armv7_pmuv1.c", + "lib/pfmlib_arm_armv6.c", + "lib/pfmlib_arm_armv8.c", + "lib/pfmlib_tx2_unc_perf_event.c", +] + +AARCH32_SRCS_LINUX = [ + "lib/pfmlib_arm_perf_event.c", +] + +AARCH64_SRCS_COMMON = [ + "lib/pfmlib_arm.c", + "lib/pfmlib_arm_armv8.c", + "lib/pfmlib_tx2_unc_perf_event.c", +] + +AARCH64_SRCS_LINUX = [ + "lib/pfmlib_arm_perf_event.c", +] + +MIPS_SRCS_COMMON = [ + "lib/pfmlib_mips.c", + "lib/pfmlib_mips_74k.c", +] + +MIPS_SRCS_LINUX = [ + "lib/pfmlib_mips_perf_event.c", +] + +POWERPC_SRCS_COMMON = [ + "lib/pfmlib_powerpc.c", + "lib/pfmlib_power4.c", + "lib/pfmlib_ppc970.c", + "lib/pfmlib_power5.c", + "lib/pfmlib_power6.c", + "lib/pfmlib_power7.c", + "lib/pfmlib_torrent.c", + "lib/pfmlib_power8.c", + "lib/pfmlib_power9.c", + "lib/pfmlib_powerpc_nest.c", +] + +POWERPC_SRCS_LINUX = [ + "lib/pfmlib_powerpc_perf_event.c", +] + +S390X_SRCS_COMMON = [ + "lib/pfmlib_s390x_cpumf.c", +] + +S390X_SRCS_LINUX = [ + "lib/pfmlib_s390x_perf_event.c", +] + +X86_64_SRCS_COMMON = [ + "lib/pfmlib_amd64.c", + "lib/pfmlib_intel_core.c", + "lib/pfmlib_intel_x86.c", + "lib/pfmlib_intel_x86_arch.c", + "lib/pfmlib_intel_atom.c", + "lib/pfmlib_intel_nhm_unc.c", + "lib/pfmlib_intel_nhm.c", + "lib/pfmlib_intel_wsm.c", + "lib/pfmlib_intel_snb.c", + "lib/pfmlib_intel_snb_unc.c", + "lib/pfmlib_intel_ivb.c", + "lib/pfmlib_intel_ivb_unc.c", + "lib/pfmlib_intel_hsw.c", + "lib/pfmlib_intel_bdw.c", + "lib/pfmlib_intel_skl.c", + "lib/pfmlib_intel_icl.c", + "lib/pfmlib_intel_rapl.c", + "lib/pfmlib_intel_snbep_unc.c", + "lib/pfmlib_intel_snbep_unc_cbo.c", + "lib/pfmlib_intel_snbep_unc_ha.c", + "lib/pfmlib_intel_snbep_unc_imc.c", + "lib/pfmlib_intel_snbep_unc_pcu.c", + "lib/pfmlib_intel_snbep_unc_qpi.c", + "lib/pfmlib_intel_snbep_unc_ubo.c", + "lib/pfmlib_intel_snbep_unc_r2pcie.c", + "lib/pfmlib_intel_snbep_unc_r3qpi.c", + "lib/pfmlib_intel_ivbep_unc_cbo.c", + "lib/pfmlib_intel_ivbep_unc_ha.c", + "lib/pfmlib_intel_ivbep_unc_imc.c", + "lib/pfmlib_intel_ivbep_unc_pcu.c", + "lib/pfmlib_intel_ivbep_unc_qpi.c", + "lib/pfmlib_intel_ivbep_unc_ubo.c", + "lib/pfmlib_intel_ivbep_unc_r2pcie.c", + "lib/pfmlib_intel_ivbep_unc_r3qpi.c", + "lib/pfmlib_intel_ivbep_unc_irp.c", + "lib/pfmlib_intel_hswep_unc_cbo.c", + "lib/pfmlib_intel_hswep_unc_ha.c", + "lib/pfmlib_intel_hswep_unc_imc.c", + "lib/pfmlib_intel_hswep_unc_pcu.c", + "lib/pfmlib_intel_hswep_unc_qpi.c", + "lib/pfmlib_intel_hswep_unc_ubo.c", + "lib/pfmlib_intel_hswep_unc_r2pcie.c", + "lib/pfmlib_intel_hswep_unc_r3qpi.c", + "lib/pfmlib_intel_hswep_unc_irp.c", + "lib/pfmlib_intel_hswep_unc_sbo.c", + "lib/pfmlib_intel_bdx_unc_cbo.c", + "lib/pfmlib_intel_bdx_unc_ubo.c", + "lib/pfmlib_intel_bdx_unc_sbo.c", + "lib/pfmlib_intel_bdx_unc_ha.c", + "lib/pfmlib_intel_bdx_unc_imc.c", + "lib/pfmlib_intel_bdx_unc_irp.c", + "lib/pfmlib_intel_bdx_unc_pcu.c", + "lib/pfmlib_intel_bdx_unc_qpi.c", + "lib/pfmlib_intel_bdx_unc_r2pcie.c", + "lib/pfmlib_intel_bdx_unc_r3qpi.c", + "lib/pfmlib_intel_skx_unc_cha.c", + "lib/pfmlib_intel_skx_unc_iio.c", + "lib/pfmlib_intel_skx_unc_imc.c", + "lib/pfmlib_intel_skx_unc_irp.c", + "lib/pfmlib_intel_skx_unc_m2m.c", + "lib/pfmlib_intel_skx_unc_m3upi.c", + "lib/pfmlib_intel_skx_unc_pcu.c", + "lib/pfmlib_intel_skx_unc_ubo.c", + "lib/pfmlib_intel_skx_unc_upi.c", + "lib/pfmlib_intel_knc.c", + "lib/pfmlib_intel_slm.c", + "lib/pfmlib_intel_tmt.c", + "lib/pfmlib_intel_knl.c", + "lib/pfmlib_intel_knl_unc_imc.c", + "lib/pfmlib_intel_knl_unc_edc.c", + "lib/pfmlib_intel_knl_unc_cha.c", + "lib/pfmlib_intel_knl_unc_m2pcie.c", + "lib/pfmlib_intel_glm.c", + "lib/pfmlib_intel_netburst.c", + "lib/pfmlib_amd64_k7.c", + "lib/pfmlib_amd64_k8.c", + "lib/pfmlib_amd64_fam10h.c", + "lib/pfmlib_amd64_fam11h.c", + "lib/pfmlib_amd64_fam12h.c", + "lib/pfmlib_amd64_fam14h.c", + "lib/pfmlib_amd64_fam15h.c", + "lib/pfmlib_amd64_fam17h.c", + "lib/pfmlib_amd64_fam16h.c", +] + +X86_SRCS_COMMON = X86_64_SRCS_COMMON + [ + "lib/pfmlib_intel_coreduo.c", + "lib/pfmlib_intel_p6.c", +] + +filegroup( + name = "cpu_srcs", + srcs = select({ + "@platforms//cpu:x86_32": X86_SRCS_COMMON, + "@platforms//cpu:x86_64": X86_64_SRCS_COMMON, + "@platforms//cpu:aarch32": AARCH32_SRCS_COMMON, + "@platforms//cpu:aarch64": AARCH64_SRCS_COMMON, + "@platforms//cpu:mips64": MIPS_SRCS_COMMON, + "@platforms//cpu:ppc32": POWERPC_SRCS_COMMON, + "@platforms//cpu:ppc64le": POWERPC_SRCS_COMMON, + "@platforms//cpu:ppc": POWERPC_SRCS_COMMON, + "@platforms//cpu:s390x": S390X_SRCS_COMMON, + "//conditions:default": [], + }), +) + +filegroup( + name = "linux_srcs", + srcs = select({ + "@platforms//cpu:aarch32": AARCH32_SRCS_LINUX, + "@platforms//cpu:aarch64": AARCH64_SRCS_LINUX, + "@platforms//cpu:mips64": MIPS_SRCS_LINUX, + "@platforms//cpu:ppc32": POWERPC_SRCS_LINUX, + "@platforms//cpu:ppc64le": POWERPC_SRCS_LINUX, + "@platforms//cpu:ppc": POWERPC_SRCS_LINUX, + "@platforms//cpu:s390x": S390X_SRCS_LINUX, + "//conditions:default": [], + }), +) + +filegroup( + name = "srcs", + srcs = [ + "lib/pfmlib_common.c", + "lib/pfmlib_perf_event.c", + "lib/pfmlib_perf_event_pmu.c", + "lib/pfmlib_perf_event_priv.h", + "lib/pfmlib_perf_event_raw.c", + "lib/pfmlib_torrent.c", + "lib/pfmlib_tx2_unc_perf_event.c", + ":cpu_srcs", + ] + select({ + "@platforms//os:linux": [":linux_srcs"], + "//conditions:default": [], + }), +) + +cc_library( + name = "pfm", + srcs = [ + ":srcs", + ], + hdrs = glob([ + "include/perfmon/*.h", + ]), + copts = [ + "-Wno-format-truncation", + "-Wno-use-after-free", + "-fPIC", + "-D_REENTRANT", + "-fvisibility=hidden", + ] + select({ + "@platforms//cpu:aarch32": ["-DCONFIG_PFMLIB_ARCH_ARM"], + "@platforms//cpu:aarch64": ["-DCONFIG_PFMLIB_ARCH_ARM64"], + "@platforms//cpu:mips64": ["-DCONFIG_PFMLIB_ARCH_MIPS"], + "@platforms//cpu:ppc32": ["-DCONFIG_PFMLIB_ARCH_POWERPC"], + "@platforms//cpu:ppc64le": ["-DCONFIG_PFMLIB_ARCH_POWERPC"], + "@platforms//cpu:ppc": ["-DCONFIG_PFMLIB_ARCH_POWERPC"], + "@platforms//cpu:s390x": ["-DCONFIG_PFMLIB_ARCH_S390X"], + "//conditions:default": [], + }), + includes = [ + "include", + "lib", + ], + strip_include_prefix = "include", + textual_hdrs = glob([ + "lib/**/*.h", + ]), + visibility = [ + "//visibility:public", + ], +) + +alias( + name = "libpfm", + actual = ":pfm", + visibility = [ + "//visibility:public", + ], +) diff --git a/tools/requirements.txt b/tools/requirements.txt index 3b3331b5af..eaef057fcb 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1 +1,2 @@ -scipy>=1.5.0 \ No newline at end of file +numpy == 2.5.1 +scipy == 1.18.0 diff --git a/tools/strip_asm.py b/tools/strip_asm.py index 9030550b43..f49a8c85ac 100755 --- a/tools/strip_asm.py +++ b/tools/strip_asm.py @@ -1,51 +1,52 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ strip_asm.py - Cleanup ASM output for the specified file """ -from argparse import ArgumentParser -import sys import os import re +import sys +from argparse import ArgumentParser + def find_used_labels(asm): found = set() - label_re = re.compile("\s*j[a-z]+\s+\.L([a-zA-Z0-9][a-zA-Z0-9_]*)") - for l in asm.splitlines(): - m = label_re.match(l) + label_re = re.compile(r"\s*j[a-z]+\s+\.L([a-zA-Z0-9][a-zA-Z0-9_]*)") + for line in asm.splitlines(): + m = label_re.match(line) if m: - found.add('.L%s' % m.group(1)) + found.add(".L%s" % m.group(1)) return found def normalize_labels(asm): decls = set() label_decl = re.compile("^[.]{0,1}L([a-zA-Z0-9][a-zA-Z0-9_]*)(?=:)") - for l in asm.splitlines(): - m = label_decl.match(l) + for line in asm.splitlines(): + m = label_decl.match(line) if m: decls.add(m.group(0)) if len(decls) == 0: return asm - needs_dot = next(iter(decls))[0] != '.' + needs_dot = next(iter(decls))[0] != "." if not needs_dot: return asm for ld in decls: - asm = re.sub("(^|\s+)" + ld + "(?=:|\s)", '\\1.' + ld, asm) + asm = re.sub(r"(^|\s+)" + ld + r"(?=:|\s)", "\\1." + ld, asm) return asm def transform_labels(asm): asm = normalize_labels(asm) used_decls = find_used_labels(asm) - new_asm = '' - label_decl = re.compile("^\.L([a-zA-Z0-9][a-zA-Z0-9_]*)(?=:)") - for l in asm.splitlines(): - m = label_decl.match(l) + new_asm = "" + label_decl = re.compile(r"^\.L([a-zA-Z0-9][a-zA-Z0-9_]*)(?=:)") + for line in asm.splitlines(): + m = label_decl.match(line) if not m or m.group(0) in used_decls: - new_asm += l - new_asm += '\n' + new_asm += line + new_asm += "\n" return new_asm @@ -53,30 +54,35 @@ def is_identifier(tk): if len(tk) == 0: return False first = tk[0] - if not first.isalpha() and first != '_': + if not first.isalpha() and first != "_": return False for i in range(1, len(tk)): c = tk[i] - if not c.isalnum() and c != '_': + if not c.isalnum() and c != "_": return False return True -def process_identifiers(l): + +def process_identifiers(line): """ process_identifiers - process all identifiers and modify them to have consistent names across all platforms; specifically across ELF and MachO. For example, MachO inserts an additional understore at the beginning of names. This function removes that. """ - parts = re.split(r'([a-zA-Z0-9_]+)', l) - new_line = '' + parts = re.split(r"([a-zA-Z0-9_]+)", line) + new_line = "" for tk in parts: - if is_identifier(tk): - if tk.startswith('__Z'): - tk = tk[1:] - elif tk.startswith('_') and len(tk) > 1 and \ - tk[1].isalpha() and tk[1] != 'Z': - tk = tk[1:] + if is_identifier(tk) and ( + tk.startswith("__Z") + or ( + tk.startswith("_") + and len(tk) > 1 + and tk[1].isalpha() + and tk[1] != "Z" + ) + ): + tk = tk[1:] new_line += tk return new_line @@ -85,65 +91,71 @@ def process_asm(asm): """ Strip the ASM of unwanted directives and lines """ - new_contents = '' + new_contents = "" asm = transform_labels(asm) # TODO: Add more things we want to remove discard_regexes = [ - re.compile("\s+\..*$"), # directive - re.compile("\s*#(NO_APP|APP)$"), #inline ASM - re.compile("\s*#.*$"), # comment line - re.compile("\s*\.globa?l\s*([.a-zA-Z_][a-zA-Z0-9$_.]*)"), #global directive - re.compile("\s*\.(string|asciz|ascii|[1248]?byte|short|word|long|quad|value|zero)"), - ] - keep_regexes = [ - + re.compile(r"\s+\..*$"), # directive + re.compile(r"\s*#(NO_APP|APP)$"), # inline ASM + re.compile(r"\s*#.*$"), # comment line + re.compile( + r"\s*\.globa?l\s*([.a-zA-Z_][a-zA-Z0-9$_.]*)" + ), # global directive + re.compile( + r"\s*\.(string|asciz|ascii|[1248]?byte|short|word|long|quad|value|zero)" + ), ] + keep_regexes: list[re.Pattern] = [] fn_label_def = re.compile("^[a-zA-Z_][a-zA-Z0-9_.]*:") - for l in asm.splitlines(): + for line in asm.splitlines(): # Remove Mach-O attribute - l = l.replace('@GOTPCREL', '') + line = line.replace("@GOTPCREL", "") add_line = True for reg in discard_regexes: - if reg.match(l) is not None: + if reg.match(line) is not None: add_line = False break for reg in keep_regexes: - if reg.match(l) is not None: + if reg.match(line) is not None: add_line = True break if add_line: - if fn_label_def.match(l) and len(new_contents) != 0: - new_contents += '\n' - l = process_identifiers(l) - new_contents += l - new_contents += '\n' + if fn_label_def.match(line) and len(new_contents) != 0: + new_contents += "\n" + line = process_identifiers(line) + new_contents += line + new_contents += "\n" return new_contents + def main(): - parser = ArgumentParser( - description='generate a stripped assembly file') + parser = ArgumentParser(description="generate a stripped assembly file") parser.add_argument( - 'input', metavar='input', type=str, nargs=1, - help='An input assembly file') + "input", + metavar="input", + type=str, + nargs=1, + help="An input assembly file", + ) parser.add_argument( - 'out', metavar='output', type=str, nargs=1, - help='The output file') - args, unknown_args = parser.parse_known_args() + "out", metavar="output", type=str, nargs=1, help="The output file" + ) + args, _ = parser.parse_known_args() input = args.input[0] output = args.out[0] if not os.path.isfile(input): - print(("ERROR: input file '%s' does not exist") % input) + print("ERROR: input file '%s' does not exist" % input) sys.exit(1) - contents = None - with open(input, 'r') as f: + + with open(input) as f: contents = f.read() new_contents = process_asm(contents) - with open(output, 'w') as f: + with open(output, "w") as f: f.write(new_contents) -if __name__ == '__main__': +if __name__ == "__main__": main() # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4