diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..fa05abe --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,199 @@ +name: Benchmarks + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: benchmarks-${{ github.ref }} + cancel-in-progress: true + +jobs: + bench-linux: + name: 'linux ${{ matrix.sys.compiler }}-${{ matrix.sys.version }} ${{ matrix.sys.flags }}' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + sys: + - { compiler: 'gcc', version: '12', flags: 'force_no_instr_set' } + - { compiler: 'gcc', version: '14', flags: 'avx' } + - { compiler: 'gcc', version: '13', flags: 'avx512' } + - { compiler: 'gcc', version: '13', flags: 'avx512pf' } + - { compiler: 'gcc', version: '13', flags: 'avx512vbmi' } + - { compiler: 'gcc', version: '14', flags: 'avx512vbmi2' } + - { compiler: 'gcc', version: '13', flags: 'avx512vnni' } + - { compiler: 'clang', version: '17', flags: 'sse3' } + - { compiler: 'clang', version: '17', flags: 'avx' } + + env: + PLATFORM: linux-${{ matrix.sys.compiler }}-${{ matrix.sys.version }}-${{ matrix.sys.flags }} + + defaults: + run: + shell: bash -l {0} + + steps: + - name: Setup GCC compiler + if: ${{ matrix.sys.compiler == 'gcc' }} + run: | + GCC_VERSION=${{ matrix.sys.version }} + sudo apt-get update + sudo apt-get --no-install-suggests --no-install-recommends install g++-$GCC_VERSION gcc-$GCC_VERSION + echo "CC=gcc-$GCC_VERSION" >> $GITHUB_ENV + echo "CXX=g++-$GCC_VERSION" >> $GITHUB_ENV + + - name: Setup Clang compiler + if: ${{ matrix.sys.compiler == 'clang' }} + run: | + LLVM_VERSION=${{ matrix.sys.version }} + sudo apt-get update + sudo apt-get --no-install-suggests --no-install-recommends install clang-$LLVM_VERSION g++ + sudo ln -sf /usr/include/asm-generic /usr/include/asm + echo "CC=clang-$LLVM_VERSION" >> $GITHUB_ENV + echo "CXX=clang++-$LLVM_VERSION" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + + - name: Set conda environment + uses: mamba-org/setup-micromamba@main + with: + environment-name: myenv + environment-file: environment-dev.yml + init-shell: bash + cache-downloads: true + + - name: Setup SDE + if: startsWith(matrix.sys.flags, 'avx512') + run: sh install_sde.sh + + - name: Configure CMake + env: + CC: ${{ env.CC }} + CXX: ${{ env.CXX }} + run: | + if [[ '${{ matrix.sys.flags }}' == 'avx' ]]; then CXX_FLAGS="$CXX_FLAGS -march=sandybridge"; fi + if [[ '${{ matrix.sys.flags }}' == 'sse3' ]]; then CXX_FLAGS="$CXX_FLAGS -march=nocona"; fi + if [[ '${{ matrix.sys.flags }}' == 'avx512' ]]; then CXX_FLAGS="$CXX_FLAGS -march=skylake-avx512"; fi + if [[ '${{ matrix.sys.flags }}' == 'avx512pf' ]]; then CXX_FLAGS="$CXX_FLAGS -march=knl"; fi + if [[ '${{ matrix.sys.flags }}' == 'avx512vbmi' ]]; then CXX_FLAGS="$CXX_FLAGS -march=cannonlake"; fi + if [[ '${{ matrix.sys.flags }}' == 'avx512vbmi2' ]]; then CXX_FLAGS="$CXX_FLAGS -march=icelake-server"; fi + if [[ '${{ matrix.sys.flags }}' == 'avx512vnni' ]]; then CXX_FLAGS="$CXX_FLAGS -march=knm"; fi + cmake -S . -B _bench_build \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=$CC \ + -DCMAKE_CXX_COMPILER=$CXX \ + -DCMAKE_CXX_FLAGS="$CXX_FLAGS" \ + -DBUILD_BENCHMARKS=ON + + - name: Build + run: ninja -C _bench_build bench_algorithms + + - name: Run benchmarks + run: | + if echo '${{ matrix.sys.flags }}' | grep -q 'avx512'; then + ./sde-external-9.48.0-2024-11-25-lin/sde64 -tgl -- \ + ./_bench_build/benchmark/bench_algorithms \ + --benchmark_repetitions=10 --benchmark_min_time=0.1s \ + --benchmark_out=benchmark-result.json + else + ./_bench_build/benchmark/bench_algorithms \ + --benchmark_repetitions=10 --benchmark_min_time=0.1s \ + --benchmark_out=benchmark-result.json + fi + + - name: Install compare tool dependencies + run: | + python3 -m venv _gbench_venv + _gbench_venv/bin/pip install --quiet -r "$CONDA_PREFIX/share/googlebenchmark/tools/requirements.txt" + + - name: Publish results to job summary + run: | + { + echo "## Benchmarks: \`$PLATFORM\`" + echo + echo '```' + _gbench_venv/bin/python "$CONDA_PREFIX/share/googlebenchmark/tools/compare.py" --no-color -a \ + filters benchmark-result.json BM_std_ BM_xsimd_ + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload benchmark artifact + uses: actions/upload-artifact@v4 + with: + name: bench-${{ env.PLATFORM }} + path: benchmark-result.json + if-no-files-found: error + + + bench-macos: + name: 'macos-${{ matrix.os }}' + runs-on: macos-${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [14, 15] + + env: + PLATFORM: macos-${{ matrix.os }} + + defaults: + run: + shell: bash -e -l {0} + + steps: + - uses: actions/checkout@v4 + + - name: Set conda environment + uses: mamba-org/setup-micromamba@main + with: + environment-name: myenv + environment-file: environment-dev.yml + init-shell: bash + cache-downloads: true + + - name: Configure CMake + run: | + cmake -S . -B _bench_build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \ + -DBUILD_BENCHMARKS=ON + + - name: Build + run: cmake --build _bench_build --target bench_algorithms --parallel 8 + + - name: Run benchmarks + run: | + ./_bench_build/benchmark/bench_algorithms \ + --benchmark_repetitions=10 --benchmark_min_time=0.1s \ + --benchmark_out=benchmark-result.json + + - name: Install compare tool dependencies + run: | + python3 -m venv _gbench_venv + _gbench_venv/bin/pip install --quiet -r "$CONDA_PREFIX/share/googlebenchmark/tools/requirements.txt" + + - name: Publish results to job summary + run: | + { + echo "## Benchmarks: \`$PLATFORM\`" + echo + echo '```' + _gbench_venv/bin/python "$CONDA_PREFIX/share/googlebenchmark/tools/compare.py" --no-color -a \ + filters benchmark-result.json BM_std_ BM_xsimd_ + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload benchmark artifact + uses: actions/upload-artifact@v4 + with: + name: bench-${{ env.PLATFORM }} + path: benchmark-result.json + if-no-files-found: error + diff --git a/CMakeLists.txt b/CMakeLists.txt index f7a6565..e6fcdf8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,12 +37,17 @@ target_compile_features(xsimd-algorithm INTERFACE cxx_std_20) target_link_libraries(xsimd-algorithm INTERFACE xsimd) OPTION(BUILD_TESTS "xsimd-algorithm test suite" OFF) +OPTION(BUILD_BENCHMARKS "xsimd-algorithm benchmarks" OFF) if(BUILD_TESTS) enable_testing() add_subdirectory(test) endif() +if(BUILD_BENCHMARKS) + add_subdirectory(benchmark) +endif() + # Installation # ============ diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt new file mode 100644 index 0000000..ff3c97f --- /dev/null +++ b/benchmark/CMakeLists.txt @@ -0,0 +1,30 @@ +############################################################################ +# Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and # +# Martin Renou # +# Copyright (c) QuantStack # +# Copyright (c) Serge Guelton # +# # +# Distributed under the terms of the BSD 3-Clause License. # +# # +# The full license is in the file LICENSE, distributed with this software. # +############################################################################ + +cmake_minimum_required(VERSION 3.8) + +project(xsimd-algorithm-benchmark) + +if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + find_package(xsimd-algorithm REQUIRED CONFIG) +endif () + +find_package(benchmark REQUIRED) + +add_executable(bench_algorithms bench_algorithms.cpp) + +target_link_libraries(bench_algorithms PRIVATE xsimd-algorithm benchmark::benchmark) + +target_compile_options(bench_algorithms PRIVATE + $<$:-O3>) + +set_target_properties(bench_algorithms PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/benchmark) diff --git a/benchmark/bench_algorithms.cpp b/benchmark/bench_algorithms.cpp new file mode 100644 index 0000000..7663979 --- /dev/null +++ b/benchmark/bench_algorithms.cpp @@ -0,0 +1,186 @@ +#include +#include +#include + +#include + +#include "xsimd_algorithm/stl/arange.hpp" +#include "xsimd_algorithm/stl/reduce.hpp" +#include "xsimd_algorithm/stl/transform.hpp" + +#define BENCH_ARGS ->Arg(kSmall)->Arg(kMedium)->Arg(kLarge) +namespace +{ + template + std::vector make_input(std::size_t n) + { + std::vector v(n); + std::iota(v.begin(), v.end(), T(1)); + return v; + } + + template + void BM_std_arange(benchmark::State& state) + { + std::vector v(static_cast(state.range(0))); + for (auto _ : state) + { + std::iota(v.begin(), v.end(), T(0)); + benchmark::DoNotOptimize(v.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); + } + + template + void BM_xsimd_arange(benchmark::State& state) + { + std::vector v(static_cast(state.range(0))); + for (auto _ : state) + { + xsimd::arange(v.begin(), v.end(), T(0), T(1)); + benchmark::DoNotOptimize(v.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); + } + + template + void BM_std_transform_unary(benchmark::State& state) + { + auto src = make_input(static_cast(state.range(0))); + std::vector dst(src.size()); + for (auto _ : state) + { + std::transform(src.begin(), src.end(), dst.begin(), + [](T x) noexcept + { return x * x; }); + benchmark::DoNotOptimize(dst.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); + } + + template + void BM_xsimd_transform_unary(benchmark::State& state) + { + auto src = make_input(static_cast(state.range(0))); + std::vector dst(src.size()); + for (auto _ : state) + { + xsimd::transform(src.begin(), src.end(), dst.begin(), + [](auto x) noexcept + { return x * x; }); + benchmark::DoNotOptimize(dst.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); + } + + template + void BM_std_transform_binary(benchmark::State& state) + { + std::size_t n = static_cast(state.range(0)); + auto a = make_input(n); + auto b = make_input(n); + std::vector dst(n); + for (auto _ : state) + { + std::transform(a.begin(), a.end(), b.begin(), dst.begin(), + [](T x, T y) noexcept + { return x + y; }); + benchmark::DoNotOptimize(dst.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); + } + + template + void BM_xsimd_transform_binary(benchmark::State& state) + { + std::size_t n = static_cast(state.range(0)); + auto a = make_input(n); + auto b = make_input(n); + std::vector dst(n); + for (auto _ : state) + { + xsimd::transform(a.begin(), a.end(), b.begin(), dst.begin(), + [](auto x, auto y) noexcept + { return x + y; }); + benchmark::DoNotOptimize(dst.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); + } + + template + void BM_std_reduce(benchmark::State& state) + { + auto v = make_input(static_cast(state.range(0))); + for (auto _ : state) + { + T result = std::reduce(v.begin(), v.end(), T(0)); + benchmark::DoNotOptimize(result); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); + } + + template + void BM_xsimd_reduce(benchmark::State& state) + { + auto v = make_input(static_cast(state.range(0))); + for (auto _ : state) + { + T result = xsimd::reduce(v.begin(), v.end(), T(0)); + benchmark::DoNotOptimize(result); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); + } + + static constexpr long long kSmall = 1LL << 12; + static constexpr long long kMedium = 1LL << 18; + static constexpr long long kLarge = 1LL << 22; + + // arange (std::iota is the scalar equivalent of xsimd::arange) + BENCHMARK_TEMPLATE(BM_std_arange, float) + BENCH_ARGS; + BENCHMARK_TEMPLATE(BM_xsimd_arange, float) + BENCH_ARGS; + BENCHMARK_TEMPLATE(BM_std_arange, double) + BENCH_ARGS; + BENCHMARK_TEMPLATE(BM_xsimd_arange, double) + BENCH_ARGS; + + // transform unary + BENCHMARK_TEMPLATE(BM_std_transform_unary, float) + BENCH_ARGS; + BENCHMARK_TEMPLATE(BM_xsimd_transform_unary, float) + BENCH_ARGS; + BENCHMARK_TEMPLATE(BM_std_transform_unary, double) + BENCH_ARGS; + BENCHMARK_TEMPLATE(BM_xsimd_transform_unary, double) + BENCH_ARGS; + + // transform binary + BENCHMARK_TEMPLATE(BM_std_transform_binary, float) + BENCH_ARGS; + BENCHMARK_TEMPLATE(BM_xsimd_transform_binary, float) + BENCH_ARGS; + BENCHMARK_TEMPLATE(BM_std_transform_binary, double) + BENCH_ARGS; + BENCHMARK_TEMPLATE(BM_xsimd_transform_binary, double) + BENCH_ARGS; + + // reduce + BENCHMARK_TEMPLATE(BM_std_reduce, float) + BENCH_ARGS; + BENCHMARK_TEMPLATE(BM_xsimd_reduce, float) + BENCH_ARGS; + BENCHMARK_TEMPLATE(BM_std_reduce, double) + BENCH_ARGS; + BENCHMARK_TEMPLATE(BM_xsimd_reduce, double) + BENCH_ARGS; + +} + +BENCHMARK_MAIN(); diff --git a/environment-dev.yml b/environment-dev.yml index 767ec13..6768058 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -5,4 +5,5 @@ dependencies: - cmake - xsimd=14.3.0 - doctest -- ninja \ No newline at end of file +- ninja +- benchmark \ No newline at end of file diff --git a/install_sde.sh b/install_sde.sh index 44b6372..5b4a3d2 100644 --- a/install_sde.sh +++ b/install_sde.sh @@ -1,3 +1,5 @@ +set -eu + #git clone https://github.com/marehr/intel-sde-downloader #cd intel-sde-downloader #pip install -r requirements.txt diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2285b23..bb3a5b9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -45,6 +45,8 @@ if(CMAKE_CXX_COMPILER_ID MATCHES Clang AND MSVC AND WIN32) # We are using clang- set(CMAKE_EXE_LINKER_FLAGS /MANIFEST:NO) endif() +if(BUILD_TESTS OR CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + set(XSIMD_ALGORITHM_TESTS main.cpp test_arange.cpp @@ -96,3 +98,5 @@ endif() if (XSIMD_ENABLE_WERROR) target_compile_options(test_xsimd_algorithm PRIVATE -Werror -Wall -DXSIMD_SKIP_ON_WERROR) endif() + +endif() # BUILD_TESTS OR standalone