From e48a9327d9ff94ae816aa21e197e29c45c060714 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 10 Sep 2026 11:56:54 +0200 Subject: [PATCH 01/27] Add map_unary --- CMakeLists.txt | 4 - include/xsimd_algorithm/builder.hpp | 176 ++++++++++++++++++++++++++++ include/xsimd_algorithm/macros.hpp | 20 ++++ test/CMakeLists.txt | 1 + test/test_builder.cpp | 78 ++++++++++++ 5 files changed, 275 insertions(+), 4 deletions(-) create mode 100644 include/xsimd_algorithm/builder.hpp create mode 100644 include/xsimd_algorithm/macros.hpp create mode 100644 test/test_builder.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f7a6565..790e26f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,10 +22,6 @@ endif() # Build # ===== -set(XSIMDALGO_HEADERS - ${XSIMDALGO_INCLUDE_DIR}/xsimd_algorithm/algorithms.hpp -) - add_library(xsimd-algorithm INTERFACE) target_include_directories(xsimd-algorithm INTERFACE diff --git a/include/xsimd_algorithm/builder.hpp b/include/xsimd_algorithm/builder.hpp new file mode 100644 index 0000000..ab6aa21 --- /dev/null +++ b/include/xsimd_algorithm/builder.hpp @@ -0,0 +1,176 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ALGORITHM_BUILDER_HPP +#define XSIMD_ALGORITHM_BUILDER_HPP + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "./macros.hpp" + +namespace xsimd::builder +{ + struct alignment_options + { + bool start_aligned = false; + bool end_aligned = false; + }; + + template + auto prev_aligned(T* ptr, std::size_t alignment) -> T* + { + assert(std::has_single_bit(alignment)); + auto const address = reinterpret_cast(ptr); + return reinterpret_cast(address & ~(alignment - 1)); + } + + template + auto next_aligned(T* ptr, std::size_t alignment) -> T* + { + assert(std::has_single_bit(alignment)); + auto const address = reinterpret_cast(ptr); + return reinterpret_cast((address + alignment - 1) & ~(alignment - 1)); + } + + template + auto bytes_to_next_aligned(T* ptr, std::size_t alignment) -> std::size_t + { + assert(std::has_single_bit(alignment)); + auto const address = reinterpret_cast(ptr); + return (alignment - (address & (alignment - 1))) & (alignment - 1); + } + + template + auto are_aliased(std::span lhs, std::span rhs) -> bool + { + // Comparing pointers from unrelated objects is unspecified, integers are not. + auto const lhs_begin = reinterpret_cast(lhs.data()); + auto const rhs_begin = reinterpret_cast(rhs.data()); + return (lhs_begin < rhs_begin + rhs.size_bytes()) && (rhs_begin < lhs_begin + lhs.size_bytes()); + } + + template < + typename Arch = xsimd::default_arch, + typename T, typename U, typename Func> + void map_unary_batch( + T const* XSIMD_RESTRICT begin, + T const* XSIMD_RESTRICT end, + U* XSIMD_RESTRICT out, + Func&& func) + { + using input_batch = xsimd::batch; + using output_batch = xsimd::batch; + + assert(begin <= end); + assert(static_cast(end - begin) <= input_batch::size); + + if (begin == end) [[unlikely]] + { + return; + } + + alignas(Arch::alignment()) T input_buffer[input_batch::size] {}; + alignas(Arch::alignment()) U output_buffer[output_batch::size]; + + const std::size_t in_count = static_cast(end - begin); + std::memcpy(input_buffer, begin, in_count * sizeof(T)); + func(input_batch::load_aligned(input_buffer)).store_aligned(output_buffer); + std::memcpy(out, output_buffer, in_count * sizeof(T)); + } + + template < + alignment_options aligned = {}, + typename Arch = xsimd::default_arch, + typename T, typename U, typename Func> + void map_unary(std::span in, std::span out, Func&& func) + { + using input_batch = xsimd::batch; + using output_batch = xsimd::batch; + + // Both sides can only be split at an element boundary. + // If input is not guarenteed aligned, we will try to align preferably the + // output (more expensive unaligned stores) or otherwise the input. + constexpr bool align_output = sizeof(U) >= sizeof(T); + constexpr bool load_is_aligned = aligned.start_aligned || !align_output; + constexpr bool store_is_aligned = aligned.start_aligned || align_output; + + assert(in.size() * sizeof(T) == out.size() * sizeof(U)); + assert(!are_aliased(in, out)); + + if (in.empty()) [[unlikely]] + { + return; + } + + auto ot = out.data(); + auto it = in.data(); + auto const end = in.data() + in.size(); + + // Input and output may not have the same alignment so it may be impossible + // to get both aligned, so we align a single side. + if constexpr (!aligned.start_aligned) + { + // The span may be too short to reach the next alignment boundary. + const auto head_bytes = std::min( + align_output ? bytes_to_next_aligned(ot, Arch::alignment()) + : bytes_to_next_aligned(it, Arch::alignment()), + in.size_bytes()); + assert(head_bytes % sizeof(T) == 0); + assert(head_bytes % sizeof(U) == 0); + + map_unary_batch(it, it + head_bytes / sizeof(T), ot, func); + it += head_bytes / sizeof(T); + ot += head_bytes / sizeof(U); + } + + // No loop-carried dependencies and no aliasing, so we leave the compiler + // to unroll the loop. + while (static_cast(end - it) >= input_batch::size) + { + input_batch x; + if constexpr (load_is_aligned) + { + x = input_batch::load_aligned(it); + } + else + { + x = input_batch::load_unaligned(it); + } + + const auto y = func(x); + if constexpr (store_is_aligned) + { + y.store_aligned(ot); + } + else + { + y.store_unaligned(ot); + } + + it += input_batch::size; + ot += output_batch::size; + } + + // Unlikely to be skipped, meant for users that know they allocate + // a multiple of the batch size. + if constexpr (!aligned.end_aligned) + { + map_unary_batch(it, end, ot, func); + } + } +} + +#endif diff --git a/include/xsimd_algorithm/macros.hpp b/include/xsimd_algorithm/macros.hpp new file mode 100644 index 0000000..341f898 --- /dev/null +++ b/include/xsimd_algorithm/macros.hpp @@ -0,0 +1,20 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ALGORITHM_MACRO_HPP +#define XSIMD_ALGORITHM_MACRO_HPP + +#if defined(_MSC_VER) && !defined(__clang__) +#define XSIMD_RESTRICT __restrict +#elif defined(__GNUC__) || defined(__clang__) +#define XSIMD_RESTRICT __restrict__ +#else +#define XSIMD_RESTRICT +#endif + +#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2285b23..b6af158 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -48,6 +48,7 @@ endif() set(XSIMD_ALGORITHM_TESTS main.cpp test_arange.cpp + test_builder.cpp test_iterator.cpp test_reduce.cpp test_transform.cpp diff --git a/test/test_builder.cpp b/test/test_builder.cpp new file mode 100644 index 0000000..6cd6236 --- /dev/null +++ b/test/test_builder.cpp @@ -0,0 +1,78 @@ +/*************************************************************************** + * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * + * Martin Renou * + * Copyright (c) QuantStack * + * Copyright (c) Serge Guelton * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#include "xsimd_algorithm/builder.hpp" + +#ifndef XSIMD_NO_SUPPORTED_ARCHITECTURE + +#include "doctest/doctest.h" + +#include +#include +#include +#include +#include + +namespace +{ + template + using aligned_vector = std::vector>; + + template + std::span as_span(std::vector const& v) + { + return std::span { v.data(), v.size() }; + } + + template + std::span as_span(std::vector& v) + { + return std::span { v.data(), v.size() }; + } + + template + aligned_vector make_arange(std::size_t size, T start = T { 0 }) + { + aligned_vector data(size); + std::iota(data.begin(), data.end(), start); + return data; + } +} + +TEST_CASE("map_unary int32 to int64") +{ + using input_type = std::int32_t; + using output_type = std::int64_t; + + // Not a multiple of the batch size, to exercise the tail. + static constexpr std::size_t input_size = 94; + static constexpr std::size_t output_size = input_size * sizeof(input_type) / sizeof(output_type); + + const auto input = make_arange(input_size); + auto output = aligned_vector(output_size); + + const auto func = [](auto const& x) + { return xsimd::widen(x + input_type { 1 })[0]; }; + + xsimd::builder::map_unary(as_span(input), as_span(output), func); + + static constexpr std::size_t in_batch_size = xsimd::batch::size; + static constexpr std::size_t out_batch_size = xsimd::batch::size; + + for (std::size_t i = 0; i < output_size; ++i) + { + const auto in_index = (i / out_batch_size) * in_batch_size + (i % out_batch_size); + CAPTURE(i); + CHECK(output[i] == static_cast(input[in_index] + 1)); + } +} + +#endif From e73d9e0d651cf45d63c6570dc7fc91a5260d049a Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 10 Sep 2026 13:54:38 +0200 Subject: [PATCH 02/27] Add sqrt and abs --- include/xsimd_algorithm/math.hpp | 41 +++++++++++ test/CMakeLists.txt | 1 + test/test_builder.cpp | 29 +------- test/test_math.cpp | 118 +++++++++++++++++++++++++++++++ test/utils.hpp | 81 +++++++++++++++++++++ 5 files changed, 243 insertions(+), 27 deletions(-) create mode 100644 include/xsimd_algorithm/math.hpp create mode 100644 test/test_math.cpp create mode 100644 test/utils.hpp diff --git a/include/xsimd_algorithm/math.hpp b/include/xsimd_algorithm/math.hpp new file mode 100644 index 0000000..88d2726 --- /dev/null +++ b/include/xsimd_algorithm/math.hpp @@ -0,0 +1,41 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ALGORITHM_MATH_HPP +#define XSIMD_ALGORITHM_MATH_HPP + +#include + +#include "./builder.hpp" + +namespace xsimd::algo +{ + template < + xsimd::builder::alignment_options aligned = {}, + typename Arch = xsimd::default_arch, + typename T> + void sqrt(std::span in, std::span out) + { + return xsimd::builder::map_unary( + in, out, [](auto x) + { return sqrt(x); }); + } + + template < + xsimd::builder::alignment_options aligned = {}, + typename Arch = xsimd::default_arch, + typename T> + void abs(std::span in, std::span out) + { + return xsimd::builder::map_unary( + in, out, [](auto x) + { return abs(x); }); + } +} + +#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b6af158..3ee3907 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -50,6 +50,7 @@ set(XSIMD_ALGORITHM_TESTS test_arange.cpp test_builder.cpp test_iterator.cpp + test_math.cpp test_reduce.cpp test_transform.cpp ) diff --git a/test/test_builder.cpp b/test/test_builder.cpp index 6cd6236..50c8808 100644 --- a/test/test_builder.cpp +++ b/test/test_builder.cpp @@ -17,35 +17,10 @@ #include #include -#include -#include -#include -namespace -{ - template - using aligned_vector = std::vector>; - - template - std::span as_span(std::vector const& v) - { - return std::span { v.data(), v.size() }; - } - - template - std::span as_span(std::vector& v) - { - return std::span { v.data(), v.size() }; - } +#include "utils.hpp" - template - aligned_vector make_arange(std::size_t size, T start = T { 0 }) - { - aligned_vector data(size); - std::iota(data.begin(), data.end(), start); - return data; - } -} +using namespace xsimd::test; TEST_CASE("map_unary int32 to int64") { diff --git a/test/test_math.cpp b/test/test_math.cpp new file mode 100644 index 0000000..45b0d99 --- /dev/null +++ b/test/test_math.cpp @@ -0,0 +1,118 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#include "xsimd_algorithm/math.hpp" + +#ifndef XSIMD_NO_SUPPORTED_ARCHITECTURE + +#include "doctest/doctest.h" + +#include +#include +#include + +#include "utils.hpp" + +using namespace xsimd::test; + +namespace +{ + template + struct sqrt_op + { + using value_type = T; + + template + static void apply(std::span in, std::span out) + { + xsimd::algo::sqrt(in, out); + } + + static T scalar(T x) + { + return std::sqrt(x); + } + + template + static std::vector input(std::size_t size) + { + return make_arange(size); + } + }; + + template + struct abs_op + { + using value_type = T; + + template + static void apply(std::span in, std::span out) + { + xsimd::algo::abs(in, out); + } + + static T scalar(T x) + { + return std::abs(x); + } + + template + static std::vector input(std::size_t size) + { + return make_arange(size, -static_cast(size) / 2); + } + }; + + template + void check_unary_math() + { + using value_type = typename Op::value_type; + // Not a multiple of the batch size, to exercise the tail. + constexpr std::size_t test_size = 94; + + const auto input = Op::template input(test_size); + auto output = std::vector(input.size()); + + Op::template apply(as_span(input), as_span(output)); + + for (std::size_t i = 0; i < input.size(); ++i) + { + CAPTURE(i); + CHECK(output[i] == doctest::Approx(Op::scalar(input[i]))); + } + } +} + +TEST_CASE_TEMPLATE( + "unary math", + Op, + sqrt_op, sqrt_op, + abs_op, abs_op) +{ + using value_type = typename Op::value_type; + + SUBCASE("aligned without header") + { + using allocator = typename aligned_vector::allocator_type; + check_unary_math(); + } + + SUBCASE("aligned with header") + { + using allocator = typename aligned_vector::allocator_type; + check_unary_math(); + } + + SUBCASE("unaligned with header") + { + using allocator = typename unaligned_vector::allocator_type; + check_unary_math(); + } +} + +#endif diff --git a/test/utils.hpp b/test/utils.hpp new file mode 100644 index 0000000..db9b9ed --- /dev/null +++ b/test/utils.hpp @@ -0,0 +1,81 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ALGORITHM_TEST_UTILS_HPP +#define XSIMD_ALGORITHM_TEST_UTILS_HPP + +#include +#include +#include +#include + +#include + +namespace xsimd::test +{ + template + using aligned_vector = std::vector>; + + /// An allocator returning memory guaranteed not to be aligned on @p Align. + /// + /// Over-allocates by @p Offset elements and shifts the returned pointer, so that the data + /// starts @p Offset * sizeof(T) bytes past an aligned address. + template + struct unaligned_allocator : private xsimd::aligned_allocator + { + static_assert((Offset * sizeof(T)) % Align != 0, "Shifted pointer would still be aligned"); + + using base_type = xsimd::aligned_allocator; + using value_type = T; + + // The non-type Align parameter defeats the default allocator_traits rebind. + template + struct rebind + { + using other = unaligned_allocator; + }; + + unaligned_allocator() = default; + + template + unaligned_allocator(unaligned_allocator const&) + { + } + + T* allocate(std::size_t n) { return base_type::allocate(n + Offset) + Offset; } + + void deallocate(T* p, std::size_t n) { base_type::deallocate(p - Offset, n + Offset); } + + friend bool operator==(unaligned_allocator const&, unaligned_allocator const&) { return true; } + }; + + template + using unaligned_vector = std::vector>; + + template + std::span as_span(std::vector const& v) + { + return std::span { v.data(), v.size() }; + } + + template + std::span as_span(std::vector& v) + { + return std::span { v.data(), v.size() }; + } + + template ::allocator_type> + std::vector make_arange(std::size_t size, T start = T { 0 }) + { + std::vector data(size); + std::iota(data.begin(), data.end(), start); + return data; + } +} + +#endif From 678777e6c3a567e6b9b66040e9a38ee041acb052 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 10 Sep 2026 15:01:57 +0200 Subject: [PATCH 03/27] Refactor data library --- CMakeLists.txt | 2 + test-utils/CMakeLists.txt | 23 +++++ .../include/xsimd_test_utils/math_data.hpp | 99 +++++++++++++++++++ .../include/xsimd_test_utils}/utils.hpp | 4 +- test/CMakeLists.txt | 4 +- test/test_builder.cpp | 20 ++-- test/test_math.cpp | 86 +++------------- 7 files changed, 150 insertions(+), 88 deletions(-) create mode 100644 test-utils/CMakeLists.txt create mode 100644 test-utils/include/xsimd_test_utils/math_data.hpp rename {test => test-utils/include/xsimd_test_utils}/utils.hpp (96%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 790e26f..85c1634 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,8 @@ target_link_libraries(xsimd-algorithm INTERFACE xsimd) OPTION(BUILD_TESTS "xsimd-algorithm test suite" OFF) +add_subdirectory(test-utils) + if(BUILD_TESTS) enable_testing() add_subdirectory(test) diff --git a/test-utils/CMakeLists.txt b/test-utils/CMakeLists.txt new file mode 100644 index 0000000..7f832e4 --- /dev/null +++ b/test-utils/CMakeLists.txt @@ -0,0 +1,23 @@ +############################################################################ +# Copyright (c) xsimd-algorithm contributors # +# # +# Distributed under the terms of the BSD 3-Clause License. # +# # +# The full license is in the file LICENSE, distributed with this software. # +############################################################################ + +cmake_minimum_required(VERSION 3.8) + +project(xsimd-algorithm-test-utils) + +if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + find_package(xsimd-algorithm REQUIRED CONFIG) +endif () + +add_library(xsimd-algorithm-test-utils INTERFACE) +add_library(xsimd::test-utils ALIAS xsimd-algorithm-test-utils) + +target_include_directories(xsimd-algorithm-test-utils INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/include) + +target_link_libraries(xsimd-algorithm-test-utils INTERFACE xsimd-algorithm) diff --git a/test-utils/include/xsimd_test_utils/math_data.hpp b/test-utils/include/xsimd_test_utils/math_data.hpp new file mode 100644 index 0000000..da79e1d --- /dev/null +++ b/test-utils/include/xsimd_test_utils/math_data.hpp @@ -0,0 +1,99 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ALGORITHM_TEST_UTILS_MATH_DATA_HPP +#define XSIMD_ALGORITHM_TEST_UTILS_MATH_DATA_HPP + +#include +#include +#include +#include +#include + +#include "xsimd_algorithm/builder.hpp" +#include "xsimd_algorithm/math.hpp" + +#include "xsimd_test_utils/utils.hpp" + +namespace xsimd::test +{ + /// Derives the scalar range application from the element-wise Derived::apply. + template + struct unary_op + { + using value_type = T; + + static void apply_range_scalar(std::span in, std::span out) + { + for (std::size_t i = 0; i < in.size(); ++i) + { + out[i] = Derived::apply(in[i]); + } + } + + template + static std::pair, std::vector> make_input_output(std::size_t size) + { + auto input = Derived::template make_input(size); + auto output = std::vector(input.size()); + return { std::move(input), std::move(output) }; + } + }; + + /******************* + * Test fixtures * + *******************/ + + template + struct sqrt_op : unary_op, T> + { + static constexpr auto name = "sqrt"; + + static T apply(T x) + { + return std::sqrt(x); + } + + template + static void apply_range_simd(std::span in, std::span out) + { + xsimd::algo::sqrt(in, out); + } + + template + static std::vector make_input(std::size_t size) + { + return make_arange(size); + } + }; + + template + struct abs_op : unary_op, T> + { + static constexpr auto name = "abs"; + + static T apply(T x) + { + return std::abs(x); + } + + template + static void apply_range_simd(std::span in, std::span out) + { + xsimd::algo::abs(in, out); + } + + template + static std::vector make_input(std::size_t size) + { + return make_arange(size, -static_cast(size) / 2); + } + }; +} + +#endif diff --git a/test/utils.hpp b/test-utils/include/xsimd_test_utils/utils.hpp similarity index 96% rename from test/utils.hpp rename to test-utils/include/xsimd_test_utils/utils.hpp index db9b9ed..692885b 100644 --- a/test/utils.hpp +++ b/test-utils/include/xsimd_test_utils/utils.hpp @@ -6,8 +6,8 @@ * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ -#ifndef XSIMD_ALGORITHM_TEST_UTILS_HPP -#define XSIMD_ALGORITHM_TEST_UTILS_HPP +#ifndef XSIMD_ALGORITHM_TEST_UTILS_UTILS_HPP +#define XSIMD_ALGORITHM_TEST_UTILS_UTILS_HPP #include #include diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3ee3907..a6d7e87 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -55,8 +55,8 @@ set(XSIMD_ALGORITHM_TESTS test_transform.cpp ) -add_executable(test_xsimd_algorithm ${XSIMD_ALGORITHM_TESTS})# ${XSIMD_ALGORITHM_HEADERS}) -target_link_libraries(test_xsimd_algorithm PRIVATE xsimd-algorithm) +add_executable(test_xsimd_algorithm ${XSIMD_ALGORITHM_TESTS}) +target_link_libraries(test_xsimd_algorithm PRIVATE xsimd-algorithm xsimd::test-utils) option(DOWNLOAD_DOCTEST OFF) find_package(doctest QUIET) diff --git a/test/test_builder.cpp b/test/test_builder.cpp index 50c8808..177716f 100644 --- a/test/test_builder.cpp +++ b/test/test_builder.cpp @@ -9,19 +9,15 @@ * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ -#include "xsimd_algorithm/builder.hpp" - -#ifndef XSIMD_NO_SUPPORTED_ARCHITECTURE - -#include "doctest/doctest.h" - #include #include -#include "utils.hpp" +#include +#include -using namespace xsimd::test; +#include "xsimd_algorithm/builder.hpp" +/// Map unary test that turned int32 into half has many int64. TEST_CASE("map_unary int32 to int64") { using input_type = std::int32_t; @@ -31,13 +27,13 @@ TEST_CASE("map_unary int32 to int64") static constexpr std::size_t input_size = 94; static constexpr std::size_t output_size = input_size * sizeof(input_type) / sizeof(output_type); - const auto input = make_arange(input_size); - auto output = aligned_vector(output_size); + const auto input = xsimd::test::make_arange(input_size); + auto output = xsimd::test::aligned_vector(output_size); const auto func = [](auto const& x) { return xsimd::widen(x + input_type { 1 })[0]; }; - xsimd::builder::map_unary(as_span(input), as_span(output), func); + xsimd::builder::map_unary(xsimd::test::as_span(input), xsimd::test::as_span(output), func); static constexpr std::size_t in_batch_size = xsimd::batch::size; static constexpr std::size_t out_batch_size = xsimd::batch::size; @@ -49,5 +45,3 @@ TEST_CASE("map_unary int32 to int64") CHECK(output[i] == static_cast(input[in_index] + 1)); } } - -#endif diff --git a/test/test_math.cpp b/test/test_math.cpp index 45b0d99..26ab5c8 100644 --- a/test/test_math.cpp +++ b/test/test_math.cpp @@ -6,84 +6,29 @@ * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ -#include "xsimd_algorithm/math.hpp" - -#ifndef XSIMD_NO_SUPPORTED_ARCHITECTURE - -#include "doctest/doctest.h" - -#include #include -#include -#include "utils.hpp" +#include -using namespace xsimd::test; +#include +#include namespace { - template - struct sqrt_op - { - using value_type = T; - - template - static void apply(std::span in, std::span out) - { - xsimd::algo::sqrt(in, out); - } - - static T scalar(T x) - { - return std::sqrt(x); - } - - template - static std::vector input(std::size_t size) - { - return make_arange(size); - } - }; - - template - struct abs_op - { - using value_type = T; - - template - static void apply(std::span in, std::span out) - { - xsimd::algo::abs(in, out); - } - - static T scalar(T x) - { - return std::abs(x); - } - - template - static std::vector input(std::size_t size) - { - return make_arange(size, -static_cast(size) / 2); - } - }; - template void check_unary_math() { - using value_type = typename Op::value_type; // Not a multiple of the batch size, to exercise the tail. constexpr std::size_t test_size = 94; - const auto input = Op::template input(test_size); - auto output = std::vector(input.size()); + auto [input, output] = Op::template make_input_output(test_size); - Op::template apply(as_span(input), as_span(output)); + Op::template apply_range_simd(xsimd::test::as_span(input), xsimd::test::as_span(output)); for (std::size_t i = 0; i < input.size(); ++i) { CAPTURE(i); - CHECK(output[i] == doctest::Approx(Op::scalar(input[i]))); + CHECK(output[i] == doctest::Approx(Op::apply(input[i]))); } } } @@ -91,28 +36,27 @@ namespace TEST_CASE_TEMPLATE( "unary math", Op, - sqrt_op, sqrt_op, - abs_op, abs_op) + xsimd::test::sqrt_op, + xsimd::test::sqrt_op, + xsimd::test::abs_op, + xsimd::test::abs_op) { using value_type = typename Op::value_type; + using aligned_allocator = typename xsimd::test::aligned_vector::allocator_type; + using unaligned_allocator = typename xsimd::test::unaligned_vector::allocator_type; SUBCASE("aligned without header") { - using allocator = typename aligned_vector::allocator_type; - check_unary_math(); + check_unary_math(); } SUBCASE("aligned with header") { - using allocator = typename aligned_vector::allocator_type; - check_unary_math(); + check_unary_math(); } SUBCASE("unaligned with header") { - using allocator = typename unaligned_vector::allocator_type; - check_unary_math(); + check_unary_math(); } } - -#endif From bb86eeecbb727504585ff004df3d7fccfe69cbaf Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 10 Sep 2026 16:24:03 +0200 Subject: [PATCH 04/27] Improve map_unary tail --- include/xsimd_algorithm/builder.hpp | 107 ++++++++++++------ include/xsimd_algorithm/math.hpp | 10 +- .../include/xsimd_test_utils/math_data.hpp | 4 +- .../include/xsimd_test_utils/math_ops.hpp | 75 ++++++++++++ 4 files changed, 158 insertions(+), 38 deletions(-) create mode 100644 test-utils/include/xsimd_test_utils/math_ops.hpp diff --git a/include/xsimd_algorithm/builder.hpp b/include/xsimd_algorithm/builder.hpp index ab6aa21..ddd90c4 100644 --- a/include/xsimd_algorithm/builder.hpp +++ b/include/xsimd_algorithm/builder.hpp @@ -23,14 +23,14 @@ namespace xsimd::builder { - struct alignment_options + struct alignment { bool start_aligned = false; bool end_aligned = false; }; template - auto prev_aligned(T* ptr, std::size_t alignment) -> T* + XSIMD_INLINE auto prev_aligned(T* ptr, std::size_t alignment) -> T* { assert(std::has_single_bit(alignment)); auto const address = reinterpret_cast(ptr); @@ -38,7 +38,7 @@ namespace xsimd::builder } template - auto next_aligned(T* ptr, std::size_t alignment) -> T* + XSIMD_INLINE auto next_aligned(T* ptr, std::size_t alignment) -> T* { assert(std::has_single_bit(alignment)); auto const address = reinterpret_cast(ptr); @@ -46,7 +46,7 @@ namespace xsimd::builder } template - auto bytes_to_next_aligned(T* ptr, std::size_t alignment) -> std::size_t + XSIMD_INLINE auto bytes_to_next_aligned(T* ptr, std::size_t alignment) -> std::size_t { assert(std::has_single_bit(alignment)); auto const address = reinterpret_cast(ptr); @@ -54,7 +54,7 @@ namespace xsimd::builder } template - auto are_aliased(std::span lhs, std::span rhs) -> bool + XSIMD_INLINE auto are_aliased(std::span lhs, std::span rhs) -> bool { // Comparing pointers from unrelated objects is unspecified, integers are not. auto const lhs_begin = reinterpret_cast(lhs.data()); @@ -62,10 +62,16 @@ namespace xsimd::builder return (lhs_begin < rhs_begin + rhs.size_bytes()) && (rhs_begin < lhs_begin + lhs.size_bytes()); } + struct unary_options + { + std::size_t unroll_factor = 4; + bool pure = false; + }; + template < typename Arch = xsimd::default_arch, typename T, typename U, typename Func> - void map_unary_batch( + XSIMD_INLINE void map_unary_batch( T const* XSIMD_RESTRICT begin, T const* XSIMD_RESTRICT end, U* XSIMD_RESTRICT out, @@ -91,11 +97,38 @@ namespace xsimd::builder std::memcpy(out, output_buffer, in_count * sizeof(T)); } + template + XSIMD_INLINE xsimd::batch load_batch(T const* ptr) + { + if constexpr (aligned) + { + return xsimd::batch::load_aligned(ptr); + } + else + { + return xsimd::batch::load_unaligned(ptr); + } + } + + template + XSIMD_INLINE void store_batch(xsimd::batch x, T* ptr) + { + if constexpr (aligned) + { + x.store_aligned(ptr); + } + else + { + x.store_unaligned(ptr); + } + } + template < - alignment_options aligned = {}, + alignment align = {}, + unary_options opts = {}, typename Arch = xsimd::default_arch, typename T, typename U, typename Func> - void map_unary(std::span in, std::span out, Func&& func) + XSIMD_INLINE void map_unary(std::span in, std::span out, Func&& func) { using input_batch = xsimd::batch; using output_batch = xsimd::batch; @@ -104,8 +137,8 @@ namespace xsimd::builder // If input is not guarenteed aligned, we will try to align preferably the // output (more expensive unaligned stores) or otherwise the input. constexpr bool align_output = sizeof(U) >= sizeof(T); - constexpr bool load_is_aligned = aligned.start_aligned || !align_output; - constexpr bool store_is_aligned = aligned.start_aligned || align_output; + constexpr bool load_is_aligned = align.start_aligned || !align_output; + constexpr bool store_is_aligned = align.start_aligned || align_output; assert(in.size() * sizeof(T) == out.size() * sizeof(U)); assert(!are_aliased(in, out)); @@ -117,11 +150,11 @@ namespace xsimd::builder auto ot = out.data(); auto it = in.data(); - auto const end = in.data() + in.size(); + auto const iend = in.data() + in.size(); // Input and output may not have the same alignment so it may be impossible // to get both aligned, so we align a single side. - if constexpr (!aligned.start_aligned) + if constexpr (!align.start_aligned) { // The span may be too short to reach the next alignment boundary. const auto head_bytes = std::min( @@ -136,39 +169,49 @@ namespace xsimd::builder ot += head_bytes / sizeof(U); } - // No loop-carried dependencies and no aliasing, so we leave the compiler - // to unroll the loop. - while (static_cast(end - it) >= input_batch::size) + // Unrolled loop processing multiple batches at a time + while (static_cast(iend - it) >= opts.unroll_factor * input_batch::size) { - input_batch x; - if constexpr (load_is_aligned) + input_batch x[opts.unroll_factor]; + for (std::size_t u = 0; u < opts.unroll_factor; ++u) { - x = input_batch::load_aligned(it); + x[u] = load_batch(it + u * input_batch::size); } - else + for (std::size_t u = 0; u < opts.unroll_factor; ++u) { - x = input_batch::load_unaligned(it); + store_batch(func(x[u]), ot + u * output_batch::size); } - const auto y = func(x); - if constexpr (store_is_aligned) - { - y.store_aligned(ot); - } - else - { - y.store_unaligned(ot); - } + it += opts.unroll_factor * input_batch::size; + ot += opts.unroll_factor * output_batch::size; + } + while (static_cast(iend - it) >= input_batch::size) + { + const auto x = load_batch(it); + store_batch(func(x), ot); it += input_batch::size; ot += output_batch::size; } // Unlikely to be skipped, meant for users that know they allocate - // a multiple of the batch size. - if constexpr (!aligned.end_aligned) + // a multiple of the batch size, such as in a local buffer + if constexpr (!align.end_aligned) { - map_unary_batch(it, end, ot, func); + auto const oend = out.data() + out.size(); + // Stepping back shifts the batch boundary, which pairs lanes differently + // than starting from the front unless both sides have the same lane count. + constexpr bool can_step_back = input_batch::size == output_batch::size; + if (can_step_back && it != iend && (in.size() >= input_batch::size)) [[likely]] + { + // Recompute overlapping data, this time starting from the end. + const auto x = load_batch(iend - input_batch::size); + store_batch(func(x), oend - output_batch::size); + } + else + { + map_unary_batch(it, iend, ot, func); + } } } } diff --git a/include/xsimd_algorithm/math.hpp b/include/xsimd_algorithm/math.hpp index 88d2726..369c940 100644 --- a/include/xsimd_algorithm/math.hpp +++ b/include/xsimd_algorithm/math.hpp @@ -16,23 +16,25 @@ namespace xsimd::algo { template < - xsimd::builder::alignment_options aligned = {}, + xsimd::builder::alignment align = {}, typename Arch = xsimd::default_arch, typename T> void sqrt(std::span in, std::span out) { - return xsimd::builder::map_unary( + constexpr builder::unary_options opts = { .unroll_factor = 4, .pure = true }; + return xsimd::builder::map_unary( in, out, [](auto x) { return sqrt(x); }); } template < - xsimd::builder::alignment_options aligned = {}, + xsimd::builder::alignment align = {}, typename Arch = xsimd::default_arch, typename T> void abs(std::span in, std::span out) { - return xsimd::builder::map_unary( + constexpr builder::unary_options opts = { .unroll_factor = 4, .pure = true }; + return xsimd::builder::map_unary( in, out, [](auto x) { return abs(x); }); } diff --git a/test-utils/include/xsimd_test_utils/math_data.hpp b/test-utils/include/xsimd_test_utils/math_data.hpp index da79e1d..09f221e 100644 --- a/test-utils/include/xsimd_test_utils/math_data.hpp +++ b/test-utils/include/xsimd_test_utils/math_data.hpp @@ -59,7 +59,7 @@ namespace xsimd::test return std::sqrt(x); } - template + template static void apply_range_simd(std::span in, std::span out) { xsimd::algo::sqrt(in, out); @@ -82,7 +82,7 @@ namespace xsimd::test return std::abs(x); } - template + template static void apply_range_simd(std::span in, std::span out) { xsimd::algo::abs(in, out); diff --git a/test-utils/include/xsimd_test_utils/math_ops.hpp b/test-utils/include/xsimd_test_utils/math_ops.hpp new file mode 100644 index 0000000..497df13 --- /dev/null +++ b/test-utils/include/xsimd_test_utils/math_ops.hpp @@ -0,0 +1,75 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ALGORITHM_TEST_UTILS_MATH_OPS_HPP +#define XSIMD_ALGORITHM_TEST_UTILS_MATH_OPS_HPP + +#include +#include +#include +#include + +#include +#include + +#include "xsimd_test_utils/utils.hpp" + +namespace xsimd::test +{ + template + struct sqrt_op + { + using value_type = T; + + static constexpr auto name = "sqrt"; + + template + static void apply(std::span in, std::span out) + { + xsimd::algo::sqrt(in, out); + } + + static T scalar(T x) + { + return std::sqrt(x); + } + + template + static std::vector input(std::size_t size) + { + return xsimd::test::make_arange(size); + } + }; + + template + struct abs_op + { + using value_type = T; + + static constexpr auto name = "abs"; + + template + static void apply(std::span in, std::span out) + { + xsimd::algo::abs(in, out); + } + + static T scalar(T x) + { + return std::abs(x); + } + + template + static std::vector input(std::size_t size) + { + return xsimd::test::make_arange(size, -static_cast(size) / 2); + } + }; +} + +#endif From 96f94d2f7a8b286149d4827627fa91a352c8a2f0 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 10 Sep 2026 17:01:22 +0200 Subject: [PATCH 05/27] Improve map_unary header --- include/xsimd_algorithm/builder.hpp | 35 ++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/include/xsimd_algorithm/builder.hpp b/include/xsimd_algorithm/builder.hpp index ddd90c4..8266227 100644 --- a/include/xsimd_algorithm/builder.hpp +++ b/include/xsimd_algorithm/builder.hpp @@ -10,6 +10,7 @@ #define XSIMD_ALGORITHM_BUILDER_HPP #include +#include #include #include #include @@ -88,13 +89,13 @@ namespace xsimd::builder return; } - alignas(Arch::alignment()) T input_buffer[input_batch::size] {}; - alignas(Arch::alignment()) U output_buffer[output_batch::size]; + alignas(Arch::alignment()) std::array input_buffer {}; + alignas(Arch::alignment()) std::array output_buffer; const std::size_t in_count = static_cast(end - begin); - std::memcpy(input_buffer, begin, in_count * sizeof(T)); - func(input_batch::load_aligned(input_buffer)).store_aligned(output_buffer); - std::memcpy(out, output_buffer, in_count * sizeof(T)); + std::memcpy(input_buffer.data(), begin, in_count * sizeof(T)); + func(input_batch::load_aligned(input_buffer.data())).store_aligned(output_buffer.data()); + std::memcpy(out, output_buffer.data(), in_count * sizeof(T)); } template @@ -140,6 +141,12 @@ namespace xsimd::builder constexpr bool load_is_aligned = align.start_aligned || !align_output; constexpr bool store_is_aligned = align.start_aligned || align_output; + // Edges can be handled by recomputing a region overlapping the aligned body, + // which is cheaper than a round trip through a scratch buffer. This requires + // func to be free of side effects, and both sides to have the same lane count + // since shifting the batch boundary otherwise pairs lanes differently. + constexpr bool can_overlap = opts.pure && (input_batch::size == output_batch::size); + assert(in.size() * sizeof(T) == out.size() * sizeof(U)); assert(!are_aliased(in, out)); @@ -164,7 +171,16 @@ namespace xsimd::builder assert(head_bytes % sizeof(T) == 0); assert(head_bytes % sizeof(U) == 0); - map_unary_batch(it, it + head_bytes / sizeof(T), ot, func); + if (can_overlap && (head_bytes != 0) && (in.size() >= input_batch::size)) + { + // Recompute the head as a full batch, the body overwrites the excess. + const auto x = load_batch(it); + store_batch(func(x), ot); + } + else + { + map_unary_batch(it, it + head_bytes / sizeof(T), ot, func); + } it += head_bytes / sizeof(T); ot += head_bytes / sizeof(U); } @@ -172,7 +188,7 @@ namespace xsimd::builder // Unrolled loop processing multiple batches at a time while (static_cast(iend - it) >= opts.unroll_factor * input_batch::size) { - input_batch x[opts.unroll_factor]; + std::array x; for (std::size_t u = 0; u < opts.unroll_factor; ++u) { x[u] = load_batch(it + u * input_batch::size); @@ -199,10 +215,7 @@ namespace xsimd::builder if constexpr (!align.end_aligned) { auto const oend = out.data() + out.size(); - // Stepping back shifts the batch boundary, which pairs lanes differently - // than starting from the front unless both sides have the same lane count. - constexpr bool can_step_back = input_batch::size == output_batch::size; - if (can_step_back && it != iend && (in.size() >= input_batch::size)) [[likely]] + if (can_overlap && (it != iend) && (in.size() >= input_batch::size)) [[likely]] { // Recompute overlapping data, this time starting from the end. const auto x = load_batch(iend - input_batch::size); From 85db5217503f885a967090fb9c825966b598cc65 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 10 Sep 2026 17:02:25 +0200 Subject: [PATCH 06/27] Add exp unary function --- include/xsimd_algorithm/math.hpp | 12 ++++++++ .../include/xsimd_test_utils/math_data.hpp | 29 +++++++++++++++++++ test/test_math.cpp | 6 ++-- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/include/xsimd_algorithm/math.hpp b/include/xsimd_algorithm/math.hpp index 369c940..139016a 100644 --- a/include/xsimd_algorithm/math.hpp +++ b/include/xsimd_algorithm/math.hpp @@ -38,6 +38,18 @@ namespace xsimd::algo in, out, [](auto x) { return abs(x); }); } + + template < + xsimd::builder::alignment align = {}, + typename Arch = xsimd::default_arch, + typename T> + void exp(std::span in, std::span out) + { + constexpr builder::unary_options opts = { .unroll_factor = 4, .pure = true }; + return xsimd::builder::map_unary( + in, out, [](auto x) + { return exp(x); }); + } } #endif diff --git a/test-utils/include/xsimd_test_utils/math_data.hpp b/test-utils/include/xsimd_test_utils/math_data.hpp index 09f221e..736fb7e 100644 --- a/test-utils/include/xsimd_test_utils/math_data.hpp +++ b/test-utils/include/xsimd_test_utils/math_data.hpp @@ -94,6 +94,35 @@ namespace xsimd::test return make_arange(size, -static_cast(size) / 2); } }; + + template + struct exp_op : unary_op, T> + { + static constexpr auto name = "exp"; + + static T apply(T x) + { + return std::exp(x); + } + + template + static void apply_range_simd(std::span in, std::span out) + { + xsimd::algo::exp(in, out); + } + + template + static std::vector make_input(std::size_t size) + { + // exp overflows past a small range, so wrap the values back into [-10, 10). + auto input = make_arange(size); + for (auto& x : input) + { + x = std::fmod(x, T { 20 }) - T { 10 }; + } + return input; + } + }; } #endif diff --git a/test/test_math.cpp b/test/test_math.cpp index 26ab5c8..fa995b8 100644 --- a/test/test_math.cpp +++ b/test/test_math.cpp @@ -15,7 +15,7 @@ namespace { - template + template void check_unary_math() { // Not a multiple of the batch size, to exercise the tail. @@ -39,7 +39,9 @@ TEST_CASE_TEMPLATE( xsimd::test::sqrt_op, xsimd::test::sqrt_op, xsimd::test::abs_op, - xsimd::test::abs_op) + xsimd::test::abs_op, + xsimd::test::exp_op, + xsimd::test::exp_op) { using value_type = typename Op::value_type; using aligned_allocator = typename xsimd::test::aligned_vector::allocator_type; From 01eb843c5e64761a743772baa02facf47e18cfd3 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 10 Sep 2026 17:04:23 +0200 Subject: [PATCH 07/27] Add benchmarks --- CMakeLists.txt | 5 ++ benchmark/CMakeLists.txt | 26 ++++++++ benchmark/bench_math.cpp | 124 +++++++++++++++++++++++++++++++++++++++ benchmark/main.cpp | 11 ++++ environment-dev.yml | 3 +- 5 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 benchmark/CMakeLists.txt create mode 100644 benchmark/bench_math.cpp create mode 100644 benchmark/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 85c1634..9f1837d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,7 @@ target_compile_features(xsimd-algorithm INTERFACE cxx_std_20) target_link_libraries(xsimd-algorithm INTERFACE xsimd) OPTION(BUILD_TESTS "xsimd-algorithm test suite" OFF) +OPTION(BUILD_BENCHMARK "xsimd-algorithm benchmark suite" OFF) add_subdirectory(test-utils) @@ -41,6 +42,10 @@ if(BUILD_TESTS) add_subdirectory(test) endif() +if(BUILD_BENCHMARK) + add_subdirectory(benchmark) +endif() + # Installation # ============ diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt new file mode 100644 index 0000000..5b70a13 --- /dev/null +++ b/benchmark/CMakeLists.txt @@ -0,0 +1,26 @@ +############################################################################ +# Copyright (c) xsimd-algorithm contributors # +# # +# Distributed under the terms of the BSD 3-Clause License. # +# # +# The full license is in the file LICENSE, distributed with this software. # +############################################################################ + +cmake_minimum_required(VERSION 3.8) + +project(xsimd-algorithm-benchmark) + +if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + find_package(xsimd-algorithm REQUIRED CONFIG) +endif () + +find_package(benchmark REQUIRED) + +set(XSIMD_ALGORITHM_BENCHMARKS + main.cpp + bench_math.cpp +) + +add_executable(benchmark_xsimd_algorithm ${XSIMD_ALGORITHM_BENCHMARKS}) +target_link_libraries(benchmark_xsimd_algorithm + PRIVATE xsimd-algorithm xsimd::test-utils benchmark::benchmark) diff --git a/benchmark/bench_math.cpp b/benchmark/bench_math.cpp new file mode 100644 index 0000000..35694c6 --- /dev/null +++ b/benchmark/bench_math.cpp @@ -0,0 +1,124 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + + +#include +#include +#include +#include +#include + +#include + +#include "xsimd_test_utils/math_data.hpp" +#include "xsimd_test_utils/utils.hpp" + +using xsimd::builder::alignment; + +namespace +{ + template + void bench_unary(benchmark::State& state, Apply apply) + { + using value_type = typename Op::value_type; + + auto const size = static_cast(state.range(0)); + auto [input, output] = Op::template make_input_output(size); + + for (auto _ : state) + { + apply(xsimd::test::as_span(input), xsimd::test::as_span(output)); + benchmark::DoNotOptimize(output.data()); + benchmark::ClobberMemory(); + } + + state.SetItemsProcessed(static_cast(state.iterations() * size)); + state.SetBytesProcessed( + static_cast(state.iterations() * size * 2 * sizeof(value_type))); + } + + /// Sizes spanning L1-resident to memory-bound, each with and without a scalar tail. + template + std::vector bench_sizes() + { + constexpr auto batch_size = static_cast(xsimd::batch::size); + + auto sizes = std::vector {}; + for (std::int64_t size : { 64, 1024, 65536, 1 << 21 }) + { + auto const whole = size - (size % batch_size); + sizes.push_back(whole); + sizes.push_back(whole + batch_size / 2 + 1); + } + return sizes; + } + + template + constexpr auto type_name() + { + if constexpr (std::is_same_v) + { + return "f32"; + } + else if constexpr (std::is_same_v) + { + return "f64"; + } + } + + template + void bench_simd(benchmark::State& state) + { + bench_unary( + state, + [](auto in, auto out) { Op::template apply_range_simd(in, out); }); + } + + template + void bench_scalar(benchmark::State& state) + { + bench_unary(state, [](auto in, auto out) { Op::apply_range_scalar(in, out); }); + } + + template + void register_bench(std::string_view variant, Bench bench_fn) + { + using value_type = typename Op::value_type; + + auto* bench = benchmark::RegisterBenchmark( + std::format("{}/{}/{}", Op::name, type_name(), variant), + bench_fn); + for (auto const size : bench_sizes()) + { + bench->Arg(size); + } + } + + template + void register_benches() + { + using value_type = typename Op::value_type; + using aligned_alloc = typename xsimd::test::aligned_vector::allocator_type; + using unaligned_alloc = typename xsimd::test::unaligned_vector::allocator_type; + + register_bench("scalar/aligned", bench_scalar); + register_bench("simd/aligned", bench_simd); + register_bench("simd/unaligned", bench_simd); + } + + bool const registered = [] + { + register_benches>(); + register_benches>(); + register_benches>(); + register_benches>(); + register_benches>(); + register_benches>(); + return true; + }(); +} diff --git a/benchmark/main.cpp b/benchmark/main.cpp new file mode 100644 index 0000000..68e814d --- /dev/null +++ b/benchmark/main.cpp @@ -0,0 +1,11 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#include + +BENCHMARK_MAIN(); diff --git a/environment-dev.yml b/environment-dev.yml index 767ec13..bdbfb7a 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -5,4 +5,5 @@ dependencies: - cmake - xsimd=14.3.0 - doctest -- ninja \ No newline at end of file +- benchmark +- ninja From b2fa5e3f548d281864890af8fff48286ae8db5e9 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Fri, 11 Sep 2026 10:30:13 +0200 Subject: [PATCH 08/27] Fixed element map_unary contract --- include/xsimd_algorithm/builder.hpp | 228 +++++++++++++++++++--------- test/test_builder.cpp | 66 ++++++-- 2 files changed, 207 insertions(+), 87 deletions(-) diff --git a/include/xsimd_algorithm/builder.hpp b/include/xsimd_algorithm/builder.hpp index 8266227..d8f45aa 100644 --- a/include/xsimd_algorithm/builder.hpp +++ b/include/xsimd_algorithm/builder.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include @@ -30,6 +31,7 @@ namespace xsimd::builder bool end_aligned = false; }; + /// Return the pointer before the input with the given alignment or itself if aligned. template XSIMD_INLINE auto prev_aligned(T* ptr, std::size_t alignment) -> T* { @@ -38,6 +40,7 @@ namespace xsimd::builder return reinterpret_cast(address & ~(alignment - 1)); } + /// Return the pointer after the input with the given alignment or itself if aligned. template XSIMD_INLINE auto next_aligned(T* ptr, std::size_t alignment) -> T* { @@ -54,6 +57,7 @@ namespace xsimd::builder return (alignment - (address & (alignment - 1))) & (alignment - 1); } + /// Check if two spans are aliasing each others (overlapping). template XSIMD_INLINE auto are_aliased(std::span lhs, std::span rhs) -> bool { @@ -69,35 +73,14 @@ namespace xsimd::builder bool pure = false; }; - template < - typename Arch = xsimd::default_arch, - typename T, typename U, typename Func> - XSIMD_INLINE void map_unary_batch( - T const* XSIMD_RESTRICT begin, - T const* XSIMD_RESTRICT end, - U* XSIMD_RESTRICT out, - Func&& func) - { - using input_batch = xsimd::batch; - using output_batch = xsimd::batch; - - assert(begin <= end); - assert(static_cast(end - begin) <= input_batch::size); - - if (begin == end) [[unlikely]] - { - return; - } - - alignas(Arch::alignment()) std::array input_buffer {}; - alignas(Arch::alignment()) std::array output_buffer; - - const std::size_t in_count = static_cast(end - begin); - std::memcpy(input_buffer.data(), begin, in_count * sizeof(T)); - func(input_batch::load_aligned(input_buffer.data())).store_aligned(output_buffer.data()); - std::memcpy(out, output_buffer.data(), in_count * sizeof(T)); - } + /// Number of batches of T spanning as many elements as one batch of the widest of T and U. + /// + /// Pairing that many batches on each side lets both sides advance by the same number of + /// elements, so a mapping stays elementwise regardless of the respective lane counts. + template + inline constexpr std::size_t batch_arity = sizeof(T) / std::min(sizeof(T), sizeof(U)); + /// Load batch wrapper with an alignment as template parameter. template XSIMD_INLINE xsimd::batch load_batch(T const* ptr) { @@ -111,6 +94,7 @@ namespace xsimd::builder } } + /// Store batch wrapper with an alignment as template parameter. template XSIMD_INLINE void store_batch(xsimd::batch x, T* ptr) { @@ -124,6 +108,113 @@ namespace xsimd::builder } } + /// Load an array of batches. + template + XSIMD_INLINE auto load_batches(T const* ptr) -> std::array, N> + { + std::array, N> x; + for (std::size_t i = 0; i < N; ++i) + { + x[i] = load_batch(ptr + i * xsimd::batch::size); + } + return x; + } + + /// Store an array of batches. + template + XSIMD_INLINE void store_batches(std::array, N> const& x, T* ptr) + { + for (std::size_t i = 0; i < N; ++i) + { + store_batch(x[i], ptr + i * xsimd::batch::size); + } + } + + namespace internal + { + template + inline constexpr bool is_array = false; + + template + inline constexpr bool is_array> = true; + + /// If an array contains only one element, return it. + template + XSIMD_INLINE auto const& unwrap_array(std::array const& x) + { + if constexpr (N == 1) + { + return x[0]; + } + else + { + return x; + } + } + + /// Wrap user function to handle ``xsimd::batch`` as 1D array. + /// + /// Transform 1D input array as batch from alogrithm functions to batch for to + /// the user function, and user batch result as 1D arrays for the algorithm + /// functions. + template + XSIMD_INLINE auto wrap_params_as_1d_arrays(Func&& func) + { + return [func = std::forward(func)](auto const&... x) + { + auto res = func(internal::unwrap_array(x)...); + if constexpr (internal::is_array) + { + return res; + } + else + { + return std::array { res }; + } + }; + } + } + + /// Map fewer elements than a full step through a scratch buffer. + template < + typename Arch = xsimd::default_arch, + typename T, typename U, typename Func> + XSIMD_INLINE void map_unary_batch( + T const* XSIMD_RESTRICT begin, + T const* XSIMD_RESTRICT end, + U* XSIMD_RESTRICT out, + Func&& func) + { + constexpr std::size_t in_arity = batch_arity; + constexpr std::size_t out_arity = batch_arity; + constexpr std::size_t step = in_arity * xsimd::batch::size; + static_assert(step == out_arity * xsimd::batch::size); + auto mapper = internal::wrap_params_as_1d_arrays(std::forward(func)); + + assert(begin <= end); + assert(static_cast(end - begin) <= step); + + if (begin == end) [[unlikely]] + { + return; + } + + alignas(Arch::alignment()) std::array input_buffer {}; + alignas(Arch::alignment()) std::array output_buffer; + + const std::size_t count = static_cast(end - begin); + std::memcpy(input_buffer.data(), begin, count * sizeof(T)); + store_batches( + mapper(load_batches(input_buffer.data())), + output_buffer.data()); + std::memcpy(out, output_buffer.data(), count * sizeof(U)); + } + + /// Apply func elementwise over in, writing as many elements to out. + /// + /// Func maps a std::array, batch_arity> to a + /// std::array, batch_arity>, both spanning the same element count. + /// When arity is one, a callback over plain batches is accepted as well. template < alignment align = {}, unary_options opts = {}, @@ -131,23 +222,21 @@ namespace xsimd::builder typename T, typename U, typename Func> XSIMD_INLINE void map_unary(std::span in, std::span out, Func&& func) { - using input_batch = xsimd::batch; - using output_batch = xsimd::batch; + constexpr std::size_t in_arity = batch_arity; + constexpr std::size_t out_arity = batch_arity; + // Elements consumed and produced by a single call to func. + constexpr std::size_t step = in_arity * xsimd::batch::size; + static_assert(step == out_arity * xsimd::batch::size); + auto mapper = internal::wrap_params_as_1d_arrays(std::forward(func)); - // Both sides can only be split at an element boundary. - // If input is not guarenteed aligned, we will try to align preferably the - // output (more expensive unaligned stores) or otherwise the input. + // Input and output may not have the same alignment so it may be impossible + // to get both aligned. We align preferably the output (more expensive + // unaligned stores) or otherwise the input. constexpr bool align_output = sizeof(U) >= sizeof(T); constexpr bool load_is_aligned = align.start_aligned || !align_output; constexpr bool store_is_aligned = align.start_aligned || align_output; - // Edges can be handled by recomputing a region overlapping the aligned body, - // which is cheaper than a round trip through a scratch buffer. This requires - // func to be free of side effects, and both sides to have the same lane count - // since shifting the batch boundary otherwise pairs lanes differently. - constexpr bool can_overlap = opts.pure && (input_batch::size == output_batch::size); - - assert(in.size() * sizeof(T) == out.size() * sizeof(U)); + assert(in.size() == out.size()); assert(!are_aliased(in, out)); if (in.empty()) [[unlikely]] @@ -159,55 +248,52 @@ namespace xsimd::builder auto it = in.data(); auto const iend = in.data() + in.size(); - // Input and output may not have the same alignment so it may be impossible - // to get both aligned, so we align a single side. if constexpr (!align.start_aligned) { // The span may be too short to reach the next alignment boundary. - const auto head_bytes = std::min( - align_output ? bytes_to_next_aligned(ot, Arch::alignment()) - : bytes_to_next_aligned(it, Arch::alignment()), - in.size_bytes()); - assert(head_bytes % sizeof(T) == 0); - assert(head_bytes % sizeof(U) == 0); - - if (can_overlap && (head_bytes != 0) && (in.size() >= input_batch::size)) + const auto head = std::min( + align_output ? bytes_to_next_aligned(ot, Arch::alignment()) / sizeof(U) + : bytes_to_next_aligned(it, Arch::alignment()) / sizeof(T), + in.size()); + + if (opts.pure && (head != 0) && (in.size() >= step)) { - // Recompute the head as a full batch, the body overwrites the excess. - const auto x = load_batch(it); - store_batch(func(x), ot); + // Recompute the head as a full step, the body overwrites the excess. + store_batches( + mapper(load_batches(it)), + ot); } else { - map_unary_batch(it, it + head_bytes / sizeof(T), ot, func); + map_unary_batch(it, it + head, ot, func); } - it += head_bytes / sizeof(T); - ot += head_bytes / sizeof(U); + it += head; + ot += head; } - // Unrolled loop processing multiple batches at a time - while (static_cast(iend - it) >= opts.unroll_factor * input_batch::size) + // Unrolled loop processing multiple steps at a time + while (static_cast(iend - it) >= opts.unroll_factor * step) { - std::array x; + std::array, in_arity>, opts.unroll_factor> x; for (std::size_t u = 0; u < opts.unroll_factor; ++u) { - x[u] = load_batch(it + u * input_batch::size); + x[u] = load_batches(it + u * step); } for (std::size_t u = 0; u < opts.unroll_factor; ++u) { - store_batch(func(x[u]), ot + u * output_batch::size); + store_batches(mapper(x[u]), ot + u * step); } - it += opts.unroll_factor * input_batch::size; - ot += opts.unroll_factor * output_batch::size; + it += opts.unroll_factor * step; + ot += opts.unroll_factor * step; } - while (static_cast(iend - it) >= input_batch::size) + while (static_cast(iend - it) >= step) { - const auto x = load_batch(it); - store_batch(func(x), ot); - it += input_batch::size; - ot += output_batch::size; + auto x = load_batches(it); + store_batches(mapper(x), ot); + it += step; + ot += step; } // Unlikely to be skipped, meant for users that know they allocate @@ -215,11 +301,11 @@ namespace xsimd::builder if constexpr (!align.end_aligned) { auto const oend = out.data() + out.size(); - if (can_overlap && (it != iend) && (in.size() >= input_batch::size)) [[likely]] + if (opts.pure && (it != iend) && (in.size() >= step)) [[likely]] { // Recompute overlapping data, this time starting from the end. - const auto x = load_batch(iend - input_batch::size); - store_batch(func(x), oend - output_batch::size); + auto x = load_batches(iend - step); + store_batches(mapper(x), oend - step); } else { diff --git a/test/test_builder.cpp b/test/test_builder.cpp index 177716f..5088ebd 100644 --- a/test/test_builder.cpp +++ b/test/test_builder.cpp @@ -1,14 +1,12 @@ -/*************************************************************************** - * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * - * Martin Renou * - * Copyright (c) QuantStack * - * Copyright (c) Serge Guelton * +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * * * * Distributed under the terms of the BSD 3-Clause License. * * * * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ +#include #include #include @@ -17,31 +15,67 @@ #include "xsimd_algorithm/builder.hpp" -/// Map unary test that turned int32 into half has many int64. +/// Map unary test where one input batch pairs with two output batches. TEST_CASE("map_unary int32 to int64") { using input_type = std::int32_t; using output_type = std::int64_t; // Not a multiple of the batch size, to exercise the tail. - static constexpr std::size_t input_size = 94; - static constexpr std::size_t output_size = input_size * sizeof(input_type) / sizeof(output_type); + static constexpr std::size_t size = 94; - const auto input = xsimd::test::make_arange(input_size); - auto output = xsimd::test::aligned_vector(output_size); + const auto input = xsimd::test::make_arange(size); + auto output = xsimd::test::aligned_vector(size); const auto func = [](auto const& x) - { return xsimd::widen(x + input_type { 1 })[0]; }; + { return xsimd::widen(x + input_type { 1 }); }; xsimd::builder::map_unary(xsimd::test::as_span(input), xsimd::test::as_span(output), func); - static constexpr std::size_t in_batch_size = xsimd::batch::size; - static constexpr std::size_t out_batch_size = xsimd::batch::size; + for (std::size_t i = 0; i < size; ++i) + { + CAPTURE(i); + CHECK(output[i] == static_cast(input[i] + 1)); + } +} + +/// Map unary test where two input batches pair with one output batch. +TEST_CASE("map_unary int64 to int32") +{ + using input_type = std::int64_t; + using output_type = std::int32_t; + using input_batch = xsimd::batch; + using output_batch = xsimd::batch; + + // Not a multiple of the batch size, to exercise the tail. + static constexpr std::size_t size = 94; + + const auto input = xsimd::test::make_arange(size); + auto output = xsimd::test::aligned_vector(size); + + // xsimd has no narrowing counterpart to widen, so truncate by keeping the low + // half of each lane, those of the first batch followed by those of the second. + struct low_halves + { + static constexpr unsigned get(unsigned i, unsigned n) + { + return (i < n / 2) ? 2 * i : n + 2 * (i - n / 2); + } + }; + + const auto func = [](std::array const& x) -> output_batch + { + return xsimd::shuffle( + xsimd::bitwise_cast(x[0] + input_type { 1 }), + xsimd::bitwise_cast(x[1] + input_type { 1 }), + xsimd::make_batch_constant()); + }; + + xsimd::builder::map_unary(xsimd::test::as_span(input), xsimd::test::as_span(output), func); - for (std::size_t i = 0; i < output_size; ++i) + for (std::size_t i = 0; i < size; ++i) { - const auto in_index = (i / out_batch_size) * in_batch_size + (i % out_batch_size); CAPTURE(i); - CHECK(output[i] == static_cast(input[in_index] + 1)); + CHECK(output[i] == static_cast(input[i] + 1)); } } From d9e97caae11e6e30ecf5f405efe12f5c649a91fc Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Mon, 14 Sep 2026 10:23:32 +0200 Subject: [PATCH 09/27] Add upcast test and benchmark --- benchmark/bench_math.cpp | 56 +++--------- benchmark/bench_utils.hpp | 91 +++++++++++++++++++ .../include/xsimd_test_utils/math_data.hpp | 87 +++++++++++------- test/test_math.cpp | 20 +++- 4 files changed, 173 insertions(+), 81 deletions(-) create mode 100644 benchmark/bench_utils.hpp diff --git a/benchmark/bench_math.cpp b/benchmark/bench_math.cpp index 35694c6..f17722e 100644 --- a/benchmark/bench_math.cpp +++ b/benchmark/bench_math.cpp @@ -6,15 +6,13 @@ * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ - #include #include #include -#include -#include #include +#include "bench_utils.hpp" #include "xsimd_test_utils/math_data.hpp" #include "xsimd_test_utils/utils.hpp" @@ -25,7 +23,7 @@ namespace template void bench_unary(benchmark::State& state, Apply apply) { - using value_type = typename Op::value_type; + using input_t = typename Op::input_t; auto const size = static_cast(state.range(0)); auto [input, output] = Op::template make_input_output(size); @@ -39,36 +37,7 @@ namespace state.SetItemsProcessed(static_cast(state.iterations() * size)); state.SetBytesProcessed( - static_cast(state.iterations() * size * 2 * sizeof(value_type))); - } - - /// Sizes spanning L1-resident to memory-bound, each with and without a scalar tail. - template - std::vector bench_sizes() - { - constexpr auto batch_size = static_cast(xsimd::batch::size); - - auto sizes = std::vector {}; - for (std::int64_t size : { 64, 1024, 65536, 1 << 21 }) - { - auto const whole = size - (size % batch_size); - sizes.push_back(whole); - sizes.push_back(whole + batch_size / 2 + 1); - } - return sizes; - } - - template - constexpr auto type_name() - { - if constexpr (std::is_same_v) - { - return "f32"; - } - else if constexpr (std::is_same_v) - { - return "f64"; - } + static_cast(state.iterations() * size * 2 * sizeof(input_t))); } template @@ -76,24 +45,26 @@ namespace { bench_unary( state, - [](auto in, auto out) { Op::template apply_range_simd(in, out); }); + [](auto in, auto out) + { Op::template apply_range_simd(in, out); }); } template void bench_scalar(benchmark::State& state) { - bench_unary(state, [](auto in, auto out) { Op::apply_range_scalar(in, out); }); + bench_unary(state, [](auto in, auto out) + { Op::apply_range_scalar(in, out); }); } template void register_bench(std::string_view variant, Bench bench_fn) { - using value_type = typename Op::value_type; + using input_t = typename Op::input_t; auto* bench = benchmark::RegisterBenchmark( - std::format("{}/{}/{}", Op::name, type_name(), variant), + std::format("{}/{}/{}", Op::name, xsimd::bench::type_name(), variant), bench_fn); - for (auto const size : bench_sizes()) + for (auto const size : xsimd::bench::bench_sizes()) { bench->Arg(size); } @@ -102,9 +73,9 @@ namespace template void register_benches() { - using value_type = typename Op::value_type; - using aligned_alloc = typename xsimd::test::aligned_vector::allocator_type; - using unaligned_alloc = typename xsimd::test::unaligned_vector::allocator_type; + using input_t = typename Op::input_t; + using aligned_alloc = typename xsimd::test::aligned_vector::allocator_type; + using unaligned_alloc = typename xsimd::test::unaligned_vector::allocator_type; register_bench("scalar/aligned", bench_scalar); register_bench("simd/aligned", bench_simd); @@ -119,6 +90,7 @@ namespace register_benches>(); register_benches>(); register_benches>(); + register_benches>(); return true; }(); } diff --git a/benchmark/bench_utils.hpp b/benchmark/bench_utils.hpp new file mode 100644 index 0000000..f7b3686 --- /dev/null +++ b/benchmark/bench_utils.hpp @@ -0,0 +1,91 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ALGORITHM_BENCHMARK_BENCH_UTILS_HPP +#define XSIMD_ALGORITHM_BENCHMARK_BENCH_UTILS_HPP + +#include +#include +#include + +#include + +namespace xsimd::bench +{ + template + std::vector bench_sizes() + { + constexpr auto batch_size = static_cast(xsimd::batch::size); + + auto sizes = std::vector {}; + for (std::int64_t size : { 64, 1024, 65536, 1 << 21 }) + { + auto const whole = size - (size % batch_size); + sizes.push_back(whole); + sizes.push_back(whole + batch_size / 2 + 1); + } + return sizes; + } + + template + constexpr auto type_name() + { + // Avoid failing compiler check such as std::is_same due + // to it not being an alias. + constexpr bool is_int = std::is_integral_v && !std::is_same_v; + constexpr bool is_sint = is_int && std::is_signed_v; + constexpr bool is_uint = is_int && std::is_unsigned_v; + + if constexpr (std::is_same_v) + { + return "bool"; + } + else if constexpr (is_sint && sizeof(T) == 1) + { + return "i8"; + } + else if constexpr (is_uint && sizeof(T) == 1) + { + return "u8"; + } + else if constexpr (is_sint && sizeof(T) == 2) + { + return "i16"; + } + else if constexpr (is_uint && sizeof(T) == 2) + { + return "u16"; + } + else if constexpr (is_sint && sizeof(T) == 4) + { + return "i32"; + } + else if constexpr (is_uint && sizeof(T) == 4) + { + return "u32"; + } + else if constexpr (is_sint && sizeof(T) == 8) + { + return "i64"; + } + else if constexpr (is_uint && sizeof(T) == 8) + { + return "u64"; + } + else if constexpr (std::is_same_v) + { + return "f32"; + } + else if constexpr (std::is_same_v) + { + return "f64"; + } + } +} + +#endif diff --git a/test-utils/include/xsimd_test_utils/math_data.hpp b/test-utils/include/xsimd_test_utils/math_data.hpp index 736fb7e..ee1d842 100644 --- a/test-utils/include/xsimd_test_utils/math_data.hpp +++ b/test-utils/include/xsimd_test_utils/math_data.hpp @@ -11,36 +11,51 @@ #include #include +#include #include #include #include #include "xsimd_algorithm/builder.hpp" -#include "xsimd_algorithm/math.hpp" #include "xsimd_test_utils/utils.hpp" namespace xsimd::test { /// Derives the scalar range application from the element-wise Derived::apply. - template + template struct unary_op { - using value_type = T; + using input_t = In; + using output_t = Out; - static void apply_range_scalar(std::span in, std::span out) + /// Allocator of output_t matching an allocator of input_t. + template + using output_allocator = typename std::allocator_traits::template rebind_alloc; + + static void apply_range_scalar(std::span in, std::span out) { for (std::size_t i = 0; i < in.size(); ++i) { - out[i] = Derived::apply(in[i]); + out[i] = Derived::apply_scalar(in[i]); } } + template + static void apply_range_simd(std::span in, std::span out) + { + constexpr builder::unary_options opts = { .unroll_factor = 4, .pure = Derived::pure }; + return xsimd::builder::map_unary( + in, out, [](auto x) + { return Derived::apply_batch(x); }); + } + template - static std::pair, std::vector> make_input_output(std::size_t size) + static auto make_input_output(std::size_t size) + -> std::pair, std::vector>> { auto input = Derived::template make_input(size); - auto output = std::vector(input.size()); + auto output = std::vector>(input.size()); return { std::move(input), std::move(output) }; } }; @@ -53,17 +68,12 @@ namespace xsimd::test struct sqrt_op : unary_op, T> { static constexpr auto name = "sqrt"; + static constexpr bool pure = true; - static T apply(T x) - { - return std::sqrt(x); - } + static auto apply_scalar(T x) { return std::sqrt(x); } - template - static void apply_range_simd(std::span in, std::span out) - { - xsimd::algo::sqrt(in, out); - } + template + static auto apply_batch(xsimd::batch x) { return xsimd::sqrt(x); } template static std::vector make_input(std::size_t size) @@ -76,17 +86,12 @@ namespace xsimd::test struct abs_op : unary_op, T> { static constexpr auto name = "abs"; + static constexpr bool pure = true; - static T apply(T x) - { - return std::abs(x); - } + static auto apply_scalar(T x) { return std::abs(x); } - template - static void apply_range_simd(std::span in, std::span out) - { - xsimd::algo::abs(in, out); - } + template + static auto apply_batch(xsimd::batch x) { return xsimd::abs(x); } template static std::vector make_input(std::size_t size) @@ -99,17 +104,12 @@ namespace xsimd::test struct exp_op : unary_op, T> { static constexpr auto name = "exp"; + static constexpr bool pure = true; - static T apply(T x) - { - return std::exp(x); - } + static auto apply_scalar(T x) { return std::exp(x); } - template - static void apply_range_simd(std::span in, std::span out) - { - xsimd::algo::exp(in, out); - } + template + static auto apply_batch(xsimd::batch x) { return xsimd::exp(x); } template static std::vector make_input(std::size_t size) @@ -123,6 +123,25 @@ namespace xsimd::test return input; } }; + + /// Sign-extend to the type with twice as many bytes, one input batch to two output batches. + template + struct widen_op : unary_op, T, xsimd::widen_t> + { + static constexpr auto name = "widen"; + static constexpr bool pure = true; + + static auto apply_scalar(T x) { return static_cast>(x); } + + template + static auto apply_batch(xsimd::batch x) { return xsimd::widen(x); } + + template + static std::vector make_input(std::size_t size) + { + return make_arange(size, -static_cast(size / 2)); + } + }; } #endif diff --git a/test/test_math.cpp b/test/test_math.cpp index fa995b8..08938ca 100644 --- a/test/test_math.cpp +++ b/test/test_math.cpp @@ -7,6 +7,7 @@ ****************************************************************************/ #include +#include #include @@ -28,7 +29,14 @@ namespace for (std::size_t i = 0; i < input.size(); ++i) { CAPTURE(i); - CHECK(output[i] == doctest::Approx(Op::apply(input[i]))); + if constexpr (std::is_floating_point_v) + { + CHECK(output[i] == doctest::Approx(Op::apply_scalar(input[i]))); + } + else + { + CHECK(output[i] == Op::apply_scalar(input[i])); + } } } } @@ -41,11 +49,13 @@ TEST_CASE_TEMPLATE( xsimd::test::abs_op, xsimd::test::abs_op, xsimd::test::exp_op, - xsimd::test::exp_op) + xsimd::test::exp_op, + xsimd::test::widen_op + ) { - using value_type = typename Op::value_type; - using aligned_allocator = typename xsimd::test::aligned_vector::allocator_type; - using unaligned_allocator = typename xsimd::test::unaligned_vector::allocator_type; + using input_t = typename Op::input_t; + using aligned_allocator = typename xsimd::test::aligned_vector::allocator_type; + using unaligned_allocator = typename xsimd::test::unaligned_vector::allocator_type; SUBCASE("aligned without header") { From 87f5e53819424fff12d028b8e76a89e6614569e4 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Mon, 14 Sep 2026 10:35:47 +0200 Subject: [PATCH 10/27] Fix defaults --- benchmark/bench_math.cpp | 4 ++-- include/xsimd_algorithm/builder.hpp | 4 ++-- include/xsimd_algorithm/math.hpp | 6 +++--- test-utils/include/xsimd_test_utils/math_data.hpp | 2 +- test-utils/include/xsimd_test_utils/math_ops.hpp | 4 ++-- test/test_math.cpp | 7 +++---- 6 files changed, 13 insertions(+), 14 deletions(-) diff --git a/benchmark/bench_math.cpp b/benchmark/bench_math.cpp index f17722e..4f8a59e 100644 --- a/benchmark/bench_math.cpp +++ b/benchmark/bench_math.cpp @@ -40,7 +40,7 @@ namespace static_cast(state.iterations() * size * 2 * sizeof(input_t))); } - template + template void bench_simd(benchmark::State& state) { bench_unary( @@ -78,7 +78,7 @@ namespace using unaligned_alloc = typename xsimd::test::unaligned_vector::allocator_type; register_bench("scalar/aligned", bench_scalar); - register_bench("simd/aligned", bench_simd); + register_bench("simd/aligned", bench_simd); register_bench("simd/unaligned", bench_simd); } diff --git a/include/xsimd_algorithm/builder.hpp b/include/xsimd_algorithm/builder.hpp index d8f45aa..387c916 100644 --- a/include/xsimd_algorithm/builder.hpp +++ b/include/xsimd_algorithm/builder.hpp @@ -216,8 +216,8 @@ namespace xsimd::builder /// std::array, batch_arity>, both spanning the same element count. /// When arity is one, a callback over plain batches is accepted as well. template < - alignment align = {}, - unary_options opts = {}, + alignment align = alignment{}, + unary_options opts = unary_options{}, typename Arch = xsimd::default_arch, typename T, typename U, typename Func> XSIMD_INLINE void map_unary(std::span in, std::span out, Func&& func) diff --git a/include/xsimd_algorithm/math.hpp b/include/xsimd_algorithm/math.hpp index 139016a..865cfa8 100644 --- a/include/xsimd_algorithm/math.hpp +++ b/include/xsimd_algorithm/math.hpp @@ -16,7 +16,7 @@ namespace xsimd::algo { template < - xsimd::builder::alignment align = {}, + xsimd::builder::alignment align = xsimd::builder::alignment{}, typename Arch = xsimd::default_arch, typename T> void sqrt(std::span in, std::span out) @@ -28,7 +28,7 @@ namespace xsimd::algo } template < - xsimd::builder::alignment align = {}, + xsimd::builder::alignment align = xsimd::builder::alignment{}, typename Arch = xsimd::default_arch, typename T> void abs(std::span in, std::span out) @@ -40,7 +40,7 @@ namespace xsimd::algo } template < - xsimd::builder::alignment align = {}, + xsimd::builder::alignment align = xsimd::builder::alignment{}, typename Arch = xsimd::default_arch, typename T> void exp(std::span in, std::span out) diff --git a/test-utils/include/xsimd_test_utils/math_data.hpp b/test-utils/include/xsimd_test_utils/math_data.hpp index ee1d842..0db5657 100644 --- a/test-utils/include/xsimd_test_utils/math_data.hpp +++ b/test-utils/include/xsimd_test_utils/math_data.hpp @@ -41,7 +41,7 @@ namespace xsimd::test } } - template + template static void apply_range_simd(std::span in, std::span out) { constexpr builder::unary_options opts = { .unroll_factor = 4, .pure = Derived::pure }; diff --git a/test-utils/include/xsimd_test_utils/math_ops.hpp b/test-utils/include/xsimd_test_utils/math_ops.hpp index 497df13..114a4cf 100644 --- a/test-utils/include/xsimd_test_utils/math_ops.hpp +++ b/test-utils/include/xsimd_test_utils/math_ops.hpp @@ -28,7 +28,7 @@ namespace xsimd::test static constexpr auto name = "sqrt"; - template + template static void apply(std::span in, std::span out) { xsimd::algo::sqrt(in, out); @@ -53,7 +53,7 @@ namespace xsimd::test static constexpr auto name = "abs"; - template + template static void apply(std::span in, std::span out) { xsimd::algo::abs(in, out); diff --git a/test/test_math.cpp b/test/test_math.cpp index 08938ca..ede90ac 100644 --- a/test/test_math.cpp +++ b/test/test_math.cpp @@ -16,7 +16,7 @@ namespace { - template + template void check_unary_math() { // Not a multiple of the batch size, to exercise the tail. @@ -50,8 +50,7 @@ TEST_CASE_TEMPLATE( xsimd::test::abs_op, xsimd::test::exp_op, xsimd::test::exp_op, - xsimd::test::widen_op - ) + xsimd::test::widen_op) { using input_t = typename Op::input_t; using aligned_allocator = typename xsimd::test::aligned_vector::allocator_type; @@ -59,7 +58,7 @@ TEST_CASE_TEMPLATE( SUBCASE("aligned without header") { - check_unary_math(); + check_unary_math(); } SUBCASE("aligned with header") From 7658e8bec73a876139315d8edf3faad706abdeff Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Mon, 14 Sep 2026 10:48:47 +0200 Subject: [PATCH 11/27] Rename test files --- benchmark/bench_math.cpp | 2 +- .../xsimd_test_utils/{math_data.hpp => map_unary_data.hpp} | 4 ++-- test/test_math.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename test-utils/include/xsimd_test_utils/{math_data.hpp => map_unary_data.hpp} (97%) diff --git a/benchmark/bench_math.cpp b/benchmark/bench_math.cpp index 4f8a59e..fe599bf 100644 --- a/benchmark/bench_math.cpp +++ b/benchmark/bench_math.cpp @@ -13,7 +13,7 @@ #include #include "bench_utils.hpp" -#include "xsimd_test_utils/math_data.hpp" +#include "xsimd_test_utils/map_unary_data.hpp" #include "xsimd_test_utils/utils.hpp" using xsimd::builder::alignment; diff --git a/test-utils/include/xsimd_test_utils/math_data.hpp b/test-utils/include/xsimd_test_utils/map_unary_data.hpp similarity index 97% rename from test-utils/include/xsimd_test_utils/math_data.hpp rename to test-utils/include/xsimd_test_utils/map_unary_data.hpp index 0db5657..be186d9 100644 --- a/test-utils/include/xsimd_test_utils/math_data.hpp +++ b/test-utils/include/xsimd_test_utils/map_unary_data.hpp @@ -6,8 +6,8 @@ * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ -#ifndef XSIMD_ALGORITHM_TEST_UTILS_MATH_DATA_HPP -#define XSIMD_ALGORITHM_TEST_UTILS_MATH_DATA_HPP +#ifndef XSIMD_ALGORITHM_TEST_UTILS_MAP_UNARY_DATA_HPP +#define XSIMD_ALGORITHM_TEST_UTILS_MAP_UNARY_DATA_HPP #include #include diff --git a/test/test_math.cpp b/test/test_math.cpp index ede90ac..5d35c5d 100644 --- a/test/test_math.cpp +++ b/test/test_math.cpp @@ -11,7 +11,7 @@ #include -#include +#include #include namespace From 68f242057e531c9b9e00df80906357dceb6724ef Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Mon, 14 Sep 2026 11:41:55 +0200 Subject: [PATCH 12/27] Remove i386 --- .github/workflows/linux.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 5df542f..29013c7 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -17,7 +17,6 @@ jobs: - { compiler: 'gcc', version: '13', flags: 'enable_xtl_complex' } - { compiler: 'gcc', version: '14', flags: 'avx' } - { compiler: 'gcc', version: '13', flags: 'avx512' } - - { compiler: 'gcc', version: '12', flags: 'i386' } - { compiler: 'gcc', version: '13', flags: 'avx512pf' } - { compiler: 'gcc', version: '13', flags: 'avx512vbmi' } - { compiler: 'gcc', version: '14', flags: 'avx512vbmi2' } @@ -34,10 +33,6 @@ jobs: GCC_VERSION=${{ matrix.sys.version }} sudo apt-get update sudo apt-get --no-install-suggests --no-install-recommends install g++-$GCC_VERSION - sudo dpkg --add-architecture i386 - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt-get update - sudo apt-get --no-install-suggests --no-install-recommends install gcc-$GCC_VERSION-multilib g++-$GCC_VERSION-multilib linux-libc-dev:i386 CC=gcc-$GCC_VERSION echo "CC=$CC" >> $GITHUB_ENV CXX=g++-$GCC_VERSION @@ -95,9 +90,6 @@ jobs: if [[ '${{ matrix.sys.flags }}' == 'avx512vnni' ]]; then CMAKE_EXTRA_ARGS="$CMAKE_EXTRA_ARGS -DTARGET_ARCH=knm" fi - if [[ '${{ matrix.sys.flags }}' == 'i386' ]]; then - CXX_FLAGS="$CXX_FLAGS -m32" - fi if [[ '${{ matrix.sys.flags }}' == 'force_no_instr_set' ]]; then : else From 3de529db4137361ee33001ee0c6df79ec6eea112 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Mon, 14 Sep 2026 12:46:45 +0200 Subject: [PATCH 13/27] Add transform benchmarks --- benchmark/bench_math.cpp | 26 ++++++++++++++----- include/xsimd_algorithm/stl/transform.hpp | 2 +- .../xsimd_test_utils/map_unary_data.hpp | 18 +++++++++---- test/test_math.cpp | 4 +-- 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/benchmark/bench_math.cpp b/benchmark/bench_math.cpp index fe599bf..a0b3ea3 100644 --- a/benchmark/bench_math.cpp +++ b/benchmark/bench_math.cpp @@ -40,20 +40,29 @@ namespace static_cast(state.iterations() * size * 2 * sizeof(input_t))); } - template - void bench_simd(benchmark::State& state) + template + void bench_map_unary(benchmark::State& state) { bench_unary( state, [](auto in, auto out) - { Op::template apply_range_simd(in, out); }); + { Op::template range_apply_map_unary(in, out); }); + } + + template + void bench_transform(benchmark::State& state) + { + bench_unary( + state, + [](auto in, auto out) + { Op::template range_apply_transform(in, out); }); } template void bench_scalar(benchmark::State& state) { bench_unary(state, [](auto in, auto out) - { Op::apply_range_scalar(in, out); }); + { Op::range_apply_scalar(in, out); }); } template @@ -77,9 +86,14 @@ namespace using aligned_alloc = typename xsimd::test::aligned_vector::allocator_type; using unaligned_alloc = typename xsimd::test::unaligned_vector::allocator_type; + // To avoid an explosion of benchmarks, we probagly want to only benchmark aligned for + // math ops, and benchmark map_unary/transform setups (alignment...) separately on a + // few ops. register_bench("scalar/aligned", bench_scalar); - register_bench("simd/aligned", bench_simd); - register_bench("simd/unaligned", bench_simd); + register_bench("simd-map/aligned", bench_map_unary); + register_bench("simd-map/unaligned", bench_map_unary); + register_bench("simd-transform/aligned", bench_map_unary); + register_bench("simd-transform/unaligned", bench_map_unary); } bool const registered = [] diff --git a/include/xsimd_algorithm/stl/transform.hpp b/include/xsimd_algorithm/stl/transform.hpp index d757ff8..2c53738 100644 --- a/include/xsimd_algorithm/stl/transform.hpp +++ b/include/xsimd_algorithm/stl/transform.hpp @@ -16,7 +16,7 @@ #include #include -#include "xsimd/xsimd.hpp" +#include namespace xsimd { diff --git a/test-utils/include/xsimd_test_utils/map_unary_data.hpp b/test-utils/include/xsimd_test_utils/map_unary_data.hpp index be186d9..b37e270 100644 --- a/test-utils/include/xsimd_test_utils/map_unary_data.hpp +++ b/test-utils/include/xsimd_test_utils/map_unary_data.hpp @@ -6,8 +6,8 @@ * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ -#ifndef XSIMD_ALGORITHM_TEST_UTILS_MAP_UNARY_DATA_HPP -#define XSIMD_ALGORITHM_TEST_UTILS_MAP_UNARY_DATA_HPP +#ifndef XSIMD_ALGORITHM_TEST_UTILS_MATH_DATA_HPP +#define XSIMD_ALGORITHM_TEST_UTILS_MATH_DATA_HPP #include #include @@ -17,6 +17,7 @@ #include #include "xsimd_algorithm/builder.hpp" +#include "xsimd_algorithm/stl/transform.hpp" #include "xsimd_test_utils/utils.hpp" @@ -33,7 +34,7 @@ namespace xsimd::test template using output_allocator = typename std::allocator_traits::template rebind_alloc; - static void apply_range_scalar(std::span in, std::span out) + static void range_apply_scalar(std::span in, std::span out) { for (std::size_t i = 0; i < in.size(); ++i) { @@ -41,8 +42,8 @@ namespace xsimd::test } } - template - static void apply_range_simd(std::span in, std::span out) + template + static void range_apply_map_unary(std::span in, std::span out) { constexpr builder::unary_options opts = { .unroll_factor = 4, .pure = Derived::pure }; return xsimd::builder::map_unary( @@ -50,6 +51,13 @@ namespace xsimd::test { return Derived::apply_batch(x); }); } + inline static void range_apply_transform(std::span in, std::span out) + { + return xsimd::transform( + in.data(), in.data() + in.size(), out.data(), [](auto x) + { return Derived::apply_batch(x); }); + } + template static auto make_input_output(std::size_t size) -> std::pair, std::vector>> diff --git a/test/test_math.cpp b/test/test_math.cpp index 5d35c5d..36caad7 100644 --- a/test/test_math.cpp +++ b/test/test_math.cpp @@ -16,7 +16,7 @@ namespace { - template + template void check_unary_math() { // Not a multiple of the batch size, to exercise the tail. @@ -24,7 +24,7 @@ namespace auto [input, output] = Op::template make_input_output(test_size); - Op::template apply_range_simd(xsimd::test::as_span(input), xsimd::test::as_span(output)); + Op::template range_apply_map_unary(xsimd::test::as_span(input), xsimd::test::as_span(output)); for (std::size_t i = 0; i < input.size(); ++i) { From ea0cbb65c0bda9af6fc8e82e187e7871c478b39c Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Mon, 14 Sep 2026 12:56:40 +0200 Subject: [PATCH 14/27] Add arch in benchmark name --- benchmark/bench_math.cpp | 27 ++++++++++--------- .../xsimd_test_utils/map_unary_data.hpp | 7 ++--- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/benchmark/bench_math.cpp b/benchmark/bench_math.cpp index a0b3ea3..d1455c9 100644 --- a/benchmark/bench_math.cpp +++ b/benchmark/bench_math.cpp @@ -40,22 +40,22 @@ namespace static_cast(state.iterations() * size * 2 * sizeof(input_t))); } - template + template void bench_map_unary(benchmark::State& state) { bench_unary( state, [](auto in, auto out) - { Op::template range_apply_map_unary(in, out); }); + { Op::template range_apply_map_unary(in, out); }); } - template + template void bench_transform(benchmark::State& state) { bench_unary( state, [](auto in, auto out) - { Op::template range_apply_transform(in, out); }); + { Op::template range_apply_transform(in, out); }); } template @@ -65,13 +65,13 @@ namespace { Op::range_apply_scalar(in, out); }); } - template + template void register_bench(std::string_view variant, Bench bench_fn) { using input_t = typename Op::input_t; auto* bench = benchmark::RegisterBenchmark( - std::format("{}/{}/{}", Op::name, xsimd::bench::type_name(), variant), + std::format("{}/{}/{}/{}", Arch::name(), Op::name, xsimd::bench::type_name(), variant), bench_fn); for (auto const size : xsimd::bench::bench_sizes()) { @@ -83,17 +83,18 @@ namespace void register_benches() { using input_t = typename Op::input_t; - using aligned_alloc = typename xsimd::test::aligned_vector::allocator_type; - using unaligned_alloc = typename xsimd::test::unaligned_vector::allocator_type; + using arch = xsimd::default_arch; + using aligned_alloc = typename xsimd::test::aligned_vector::allocator_type; + using unaligned_alloc = typename xsimd::test::unaligned_vector::allocator_type; // To avoid an explosion of benchmarks, we probagly want to only benchmark aligned for // math ops, and benchmark map_unary/transform setups (alignment...) separately on a // few ops. - register_bench("scalar/aligned", bench_scalar); - register_bench("simd-map/aligned", bench_map_unary); - register_bench("simd-map/unaligned", bench_map_unary); - register_bench("simd-transform/aligned", bench_map_unary); - register_bench("simd-transform/unaligned", bench_map_unary); + register_bench("scalar/aligned", bench_scalar); + register_bench("simd-map/aligned", bench_map_unary); + register_bench("simd-map/unaligned", bench_map_unary); + register_bench("simd-transform/aligned", bench_map_unary); + register_bench("simd-transform/unaligned", bench_map_unary); } bool const registered = [] diff --git a/test-utils/include/xsimd_test_utils/map_unary_data.hpp b/test-utils/include/xsimd_test_utils/map_unary_data.hpp index b37e270..3fb6d3f 100644 --- a/test-utils/include/xsimd_test_utils/map_unary_data.hpp +++ b/test-utils/include/xsimd_test_utils/map_unary_data.hpp @@ -42,18 +42,19 @@ namespace xsimd::test } } - template + template static void range_apply_map_unary(std::span in, std::span out) { constexpr builder::unary_options opts = { .unroll_factor = 4, .pure = Derived::pure }; - return xsimd::builder::map_unary( + return xsimd::builder::map_unary( in, out, [](auto x) { return Derived::apply_batch(x); }); } + template inline static void range_apply_transform(std::span in, std::span out) { - return xsimd::transform( + return xsimd::transform( in.data(), in.data() + in.size(), out.data(), [](auto x) { return Derived::apply_batch(x); }); } From 9eefd0b32d59428d9b599384a1dae7bd8f6988db Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Mon, 14 Sep 2026 15:54:34 +0200 Subject: [PATCH 15/27] Fix benchmarks --- benchmark/bench_math.cpp | 7 +++++-- .../include/xsimd_test_utils/map_unary_data.hpp | 14 ++++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/benchmark/bench_math.cpp b/benchmark/bench_math.cpp index d1455c9..2eed70c 100644 --- a/benchmark/bench_math.cpp +++ b/benchmark/bench_math.cpp @@ -93,8 +93,11 @@ namespace register_bench("scalar/aligned", bench_scalar); register_bench("simd-map/aligned", bench_map_unary); register_bench("simd-map/unaligned", bench_map_unary); - register_bench("simd-transform/aligned", bench_map_unary); - register_bench("simd-transform/unaligned", bench_map_unary); + if constexpr (std::is_same_v) + { + register_bench("simd-transform/aligned", bench_transform); + register_bench("simd-transform/unaligned", bench_transform); + } } bool const registered = [] diff --git a/test-utils/include/xsimd_test_utils/map_unary_data.hpp b/test-utils/include/xsimd_test_utils/map_unary_data.hpp index 3fb6d3f..c5782b4 100644 --- a/test-utils/include/xsimd_test_utils/map_unary_data.hpp +++ b/test-utils/include/xsimd_test_utils/map_unary_data.hpp @@ -55,8 +55,18 @@ namespace xsimd::test inline static void range_apply_transform(std::span in, std::span out) { return xsimd::transform( - in.data(), in.data() + in.size(), out.data(), [](auto x) - { return Derived::apply_batch(x); }); + in.data(), in.data() + in.size(), out.data(), + [](T x) + { + if constexpr (xsimd::is_batch::value) + { + return Derived::apply_batch(x); + } + else + { + return Derived::apply_scalar(x); + } + }); } template From 2afa6a51fcfe37a90d5bd81599e1db122dd600d8 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Wed, 16 Sep 2026 10:43:29 +0200 Subject: [PATCH 16/27] Split benchmarks --- benchmark/CMakeLists.txt | 1 + benchmark/bench_map_unary.cpp | 54 ++++++++++++++++++ benchmark/bench_math.cpp | 100 ++++++++-------------------------- benchmark/map_unary_utils.hpp | 86 +++++++++++++++++++++++++++++ 4 files changed, 165 insertions(+), 76 deletions(-) create mode 100644 benchmark/bench_map_unary.cpp create mode 100644 benchmark/map_unary_utils.hpp diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 5b70a13..ef4e231 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -19,6 +19,7 @@ find_package(benchmark REQUIRED) set(XSIMD_ALGORITHM_BENCHMARKS main.cpp bench_math.cpp + bench_map_unary.cpp ) add_executable(benchmark_xsimd_algorithm ${XSIMD_ALGORITHM_BENCHMARKS}) diff --git a/benchmark/bench_map_unary.cpp b/benchmark/bench_map_unary.cpp new file mode 100644 index 0000000..e76f701 --- /dev/null +++ b/benchmark/bench_map_unary.cpp @@ -0,0 +1,54 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#include + +#include + +#include "map_unary_utils.hpp" +#include "xsimd_test_utils/map_unary_data.hpp" +#include "xsimd_test_utils/utils.hpp" + +namespace +{ + using xsimd::bench::bench_map_unary; + using xsimd::bench::bench_scalar; + using xsimd::bench::bench_transform; + using xsimd::bench::register_bench; + using xsimd::builder::alignment; + + template + void register_benches() + { + using input_t = typename Op::input_t; + using arch = xsimd::default_arch; + using aligned_alloc = typename xsimd::test::aligned_vector::allocator_type; + using unaligned_alloc = typename xsimd::test::unaligned_vector::allocator_type; + + register_bench("aligned/scalar", bench_scalar); + register_bench("aligned/simd/map:header+trailer", bench_map_unary); + register_bench("aligned/simd/map:trailer", bench_map_unary); + register_bench("aligned/simd/transform:header+trailer", bench_transform); + + register_bench("unaligned/scalar", bench_scalar); + register_bench("unaligned/simd/map:header+trailer", bench_map_unary); + register_bench("unaligned/simd/map:trailer", bench_map_unary); + register_bench("unaligned/simd/transform:header+trailer", bench_transform); + } + + bool const registered = [] + { + register_benches>(); + register_benches>(); + register_benches>(); + register_benches>(); + register_benches>(); + register_benches>(); + return true; + }(); +} diff --git a/benchmark/bench_math.cpp b/benchmark/bench_math.cpp index 2eed70c..6b2e1dc 100644 --- a/benchmark/bench_math.cpp +++ b/benchmark/bench_math.cpp @@ -7,97 +7,43 @@ ****************************************************************************/ #include -#include -#include #include -#include "bench_utils.hpp" +#include "map_unary_utils.hpp" #include "xsimd_test_utils/map_unary_data.hpp" #include "xsimd_test_utils/utils.hpp" -using xsimd::builder::alignment; - namespace { - template - void bench_unary(benchmark::State& state, Apply apply) - { - using input_t = typename Op::input_t; - - auto const size = static_cast(state.range(0)); - auto [input, output] = Op::template make_input_output(size); - - for (auto _ : state) - { - apply(xsimd::test::as_span(input), xsimd::test::as_span(output)); - benchmark::DoNotOptimize(output.data()); - benchmark::ClobberMemory(); - } - - state.SetItemsProcessed(static_cast(state.iterations() * size)); - state.SetBytesProcessed( - static_cast(state.iterations() * size * 2 * sizeof(input_t))); - } - - template - void bench_map_unary(benchmark::State& state) - { - bench_unary( - state, - [](auto in, auto out) - { Op::template range_apply_map_unary(in, out); }); - } - - template - void bench_transform(benchmark::State& state) - { - bench_unary( - state, - [](auto in, auto out) - { Op::template range_apply_transform(in, out); }); - } - - template - void bench_scalar(benchmark::State& state) - { - bench_unary(state, [](auto in, auto out) - { Op::range_apply_scalar(in, out); }); - } - - template - void register_bench(std::string_view variant, Bench bench_fn) - { - using input_t = typename Op::input_t; - - auto* bench = benchmark::RegisterBenchmark( - std::format("{}/{}/{}/{}", Arch::name(), Op::name, xsimd::bench::type_name(), variant), - bench_fn); - for (auto const size : xsimd::bench::bench_sizes()) - { - bench->Arg(size); - } - } - + using xsimd::bench::bench_map_unary; + using xsimd::bench::bench_scalar; + using xsimd::bench::register_bench; + using xsimd::builder::alignment; + + /// Register math benchmarks. + /// + /// To avoid an explosion of benchmarks, we only add a simple aligned benchmark. + /// This will let us know the performance of the xsimd wrappers. + /// See bench_map_unary for benchmarks on the different flavor of mapping, alignment, + /// headers and trailers. + /// + /// This benchmark aims to test raw the performance of intrinsic, unrelated to + /// how they are iterated on (alignment, memory etc). To do so, they aim to stay + /// in L1 cache. template void register_benches() { using input_t = typename Op::input_t; using arch = xsimd::default_arch; using aligned_alloc = typename xsimd::test::aligned_vector::allocator_type; - using unaligned_alloc = typename xsimd::test::unaligned_vector::allocator_type; - // To avoid an explosion of benchmarks, we probagly want to only benchmark aligned for - // math ops, and benchmark map_unary/transform setups (alignment...) separately on a - // few ops. - register_bench("scalar/aligned", bench_scalar); - register_bench("simd-map/aligned", bench_map_unary); - register_bench("simd-map/unaligned", bench_map_unary); - if constexpr (std::is_same_v) - { - register_bench("simd-transform/aligned", bench_transform); - register_bench("simd-transform/unaligned", bench_transform); - } + register_bench( + "hot/scalar", bench_scalar, /* sizes = */ { 1024 }); + register_bench( + "hot/simd", + bench_map_unary, + /* sizes = */ { 1024 }); } bool const registered = [] @@ -108,6 +54,8 @@ namespace register_benches>(); register_benches>(); register_benches>(); + register_benches>(); + register_benches>(); register_benches>(); return true; }(); diff --git a/benchmark/map_unary_utils.hpp b/benchmark/map_unary_utils.hpp new file mode 100644 index 0000000..07c8666 --- /dev/null +++ b/benchmark/map_unary_utils.hpp @@ -0,0 +1,86 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#include +#include +#include +#include +#include + +#include + +#include "bench_utils.hpp" +#include "xsimd_algorithm/builder.hpp" +#include "xsimd_test_utils/utils.hpp" + +namespace xsimd::bench +{ + using xsimd::builder::alignment; + + template + void bench_unary(benchmark::State& state, Apply apply) + { + using input_t = typename Op::input_t; + + auto const size = static_cast(state.range(0)); + auto [input, output] = Op::template make_input_output(size); + + for (auto _ : state) + { + apply(xsimd::test::as_span(input), xsimd::test::as_span(output)); + benchmark::DoNotOptimize(output.data()); + benchmark::ClobberMemory(); + } + + state.SetItemsProcessed(static_cast(state.iterations() * size)); + state.SetBytesProcessed( + static_cast(state.iterations() * size * 2 * sizeof(input_t))); + } + + template + void bench_map_unary(benchmark::State& state) + { + bench_unary( + state, + [](auto in, auto out) + { Op::template range_apply_map_unary(in, out); }); + } + + template + void bench_transform(benchmark::State& state) + { + bench_unary( + state, + [](auto in, auto out) + { Op::template range_apply_transform(in, out); }); + } + + template + void bench_scalar(benchmark::State& state) + { + bench_unary(state, [](auto in, auto out) + { Op::range_apply_scalar(in, out); }); + } + + template + void register_bench( + std::string_view variant, + Bench bench_fn, + std::vector const& sizes = xsimd::bench::bench_sizes()) + { + using input_t = typename Op::input_t; + + auto* bench = benchmark::RegisterBenchmark( + std::format("{}/{}/{}/{}", Arch::name(), Op::name, xsimd::bench::type_name(), variant), + bench_fn); + for (auto const size : sizes) + { + bench->Arg(size); + } + } +} From f25fea35a7d63728b76e69db6019549d8a459434 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Wed, 16 Sep 2026 11:42:11 +0200 Subject: [PATCH 17/27] More explicit unary benchmarks --- benchmark/bench_map_unary.cpp | 52 ++++++++++++++++--- benchmark/map_unary_utils.hpp | 10 +++- .../xsimd_test_utils/map_unary_data.hpp | 9 +++- 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/benchmark/bench_map_unary.cpp b/benchmark/bench_map_unary.cpp index e76f701..9919da0 100644 --- a/benchmark/bench_map_unary.cpp +++ b/benchmark/bench_map_unary.cpp @@ -11,6 +11,7 @@ #include #include "map_unary_utils.hpp" +#include "xsimd_algorithm/builder.hpp" #include "xsimd_test_utils/map_unary_data.hpp" #include "xsimd_test_utils/utils.hpp" @@ -21,6 +22,7 @@ namespace using xsimd::bench::bench_transform; using xsimd::bench::register_bench; using xsimd::builder::alignment; + using xsimd::builder::unary_options; template void register_benches() @@ -30,15 +32,53 @@ namespace using aligned_alloc = typename xsimd::test::aligned_vector::allocator_type; using unaligned_alloc = typename xsimd::test::unaligned_vector::allocator_type; + constexpr auto noalign = alignment {}; + constexpr auto noopts = unary_options { .unroll_factor = 1, .pure = false }; + register_bench("aligned/scalar", bench_scalar); - register_bench("aligned/simd/map:header+trailer", bench_map_unary); - register_bench("aligned/simd/map:trailer", bench_map_unary); - register_bench("aligned/simd/transform:header+trailer", bench_transform); + register_bench("aligned/simd/transform", bench_transform); + register_bench( + "aligned/simd/map", + bench_map_unary); + register_bench( + "aligned/simd/map:pure", + bench_map_unary< + Op, aligned_alloc, arch, + noalign, unary_options { .unroll_factor = 1, .pure = true }>); + register_bench( + "aligned/simd/map:unroll4", + bench_map_unary< + Op, aligned_alloc, arch, noalign, unary_options { .unroll_factor = 4 }>); + register_bench( + "aligned/simd/map:noheader", + bench_map_unary< + Op, aligned_alloc, arch, + alignment { .start_aligned = true }, noopts>); + register_bench( + "aligned/simd/map:noheader+pure+unroll4", + bench_map_unary< + Op, aligned_alloc, arch, + alignment { .start_aligned = true }, unary_options { .unroll_factor = 4, .pure = true }>); register_bench("unaligned/scalar", bench_scalar); - register_bench("unaligned/simd/map:header+trailer", bench_map_unary); - register_bench("unaligned/simd/map:trailer", bench_map_unary); - register_bench("unaligned/simd/transform:header+trailer", bench_transform); + register_bench("unaligned/simd/transform", bench_transform); + register_bench( + "unaligned/simd/map", + bench_map_unary); + register_bench( + "unaligned/simd/map:pure", + bench_map_unary< + Op, unaligned_alloc, arch, + noalign, unary_options { .unroll_factor = 1, .pure = true }>); + register_bench( + "unaligned/simd/map:unroll4", + bench_map_unary< + Op, unaligned_alloc, arch, noalign, unary_options { .unroll_factor = 4 }>); + register_bench( + "unaligned/simd/map:pure+unroll4", + bench_map_unary< + Op, unaligned_alloc, arch, + noalign, unary_options { .unroll_factor = 4, .pure = true }>); } bool const registered = [] diff --git a/benchmark/map_unary_utils.hpp b/benchmark/map_unary_utils.hpp index 07c8666..d668863 100644 --- a/benchmark/map_unary_utils.hpp +++ b/benchmark/map_unary_utils.hpp @@ -21,6 +21,7 @@ namespace xsimd::bench { using xsimd::builder::alignment; + using xsimd::builder::unary_options; template void bench_unary(benchmark::State& state, Apply apply) @@ -42,13 +43,18 @@ namespace xsimd::bench static_cast(state.iterations() * size * 2 * sizeof(input_t))); } - template + template < + typename Op, + typename Alloc, + typename Arch, + alignment aligned = alignment {}, + unary_options opts = unary_options {}> void bench_map_unary(benchmark::State& state) { bench_unary( state, [](auto in, auto out) - { Op::template range_apply_map_unary(in, out); }); + { Op::template range_apply_map_unary(in, out); }); } template diff --git a/test-utils/include/xsimd_test_utils/map_unary_data.hpp b/test-utils/include/xsimd_test_utils/map_unary_data.hpp index c5782b4..8a1bdba 100644 --- a/test-utils/include/xsimd_test_utils/map_unary_data.hpp +++ b/test-utils/include/xsimd_test_utils/map_unary_data.hpp @@ -23,6 +23,9 @@ namespace xsimd::test { + using xsimd::builder::alignment; + using xsimd::builder::unary_options; + /// Derives the scalar range application from the element-wise Derived::apply. template struct unary_op @@ -42,10 +45,12 @@ namespace xsimd::test } } - template + template < + alignment aligned = alignment {}, + unary_options opts = unary_options {}, + typename Arch = xsimd::default_arch> static void range_apply_map_unary(std::span in, std::span out) { - constexpr builder::unary_options opts = { .unroll_factor = 4, .pure = Derived::pure }; return xsimd::builder::map_unary( in, out, [](auto x) { return Derived::apply_batch(x); }); From 656395fcbe37c235c7a1891946e5d69201ec9bc0 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Wed, 16 Sep 2026 15:52:24 +0200 Subject: [PATCH 18/27] Refactor map_unary with helpers --- include/xsimd_algorithm/builder.hpp | 172 +++++++++++++++------------- 1 file changed, 90 insertions(+), 82 deletions(-) diff --git a/include/xsimd_algorithm/builder.hpp b/include/xsimd_algorithm/builder.hpp index 387c916..8eb00d4 100644 --- a/include/xsimd_algorithm/builder.hpp +++ b/include/xsimd_algorithm/builder.hpp @@ -73,13 +73,6 @@ namespace xsimd::builder bool pure = false; }; - /// Number of batches of T spanning as many elements as one batch of the widest of T and U. - /// - /// Pairing that many batches on each side lets both sides advance by the same number of - /// elements, so a mapping stays elementwise regardless of the respective lane counts. - template - inline constexpr std::size_t batch_arity = sizeof(T) / std::min(sizeof(T), sizeof(U)); - /// Load batch wrapper with an alignment as template parameter. template XSIMD_INLINE xsimd::batch load_batch(T const* ptr) @@ -108,28 +101,6 @@ namespace xsimd::builder } } - /// Load an array of batches. - template - XSIMD_INLINE auto load_batches(T const* ptr) -> std::array, N> - { - std::array, N> x; - for (std::size_t i = 0; i < N; ++i) - { - x[i] = load_batch(ptr + i * xsimd::batch::size); - } - return x; - } - - /// Store an array of batches. - template - XSIMD_INLINE void store_batches(std::array, N> const& x, T* ptr) - { - for (std::size_t i = 0; i < N; ++i) - { - store_batch(x[i], ptr + i * xsimd::batch::size); - } - } - namespace internal { template @@ -154,7 +125,7 @@ namespace xsimd::builder /// Wrap user function to handle ``xsimd::batch`` as 1D array. /// - /// Transform 1D input array as batch from alogrithm functions to batch for to + /// Transform 1D input array as batch from algorithm functions to batch for to /// the user function, and user batch result as 1D arrays for the algorithm /// functions. template @@ -175,38 +146,79 @@ namespace xsimd::builder } } + template + struct map_helper + { + using out_t = Out; + using in_t = In; + static constexpr std::size_t min_elem_size = std::min(sizeof(out_t), sizeof(in_t)); + + /// Number of batches of T spanning as many elements as one batch of the widest element. + /// + /// Pairing that many batches on all side lets both sides advance by the same number of + /// elements, so a mapping stays elementwise regardless of the respective lane counts. + template + static constexpr std::size_t batch_arity() + { + return sizeof(T) / min_elem_size; + } + + template + using batch_array = std::array, batch_arity()>; + + static constexpr std::size_t out_arity = batch_arity(); + static constexpr std::size_t in_arity = batch_arity(); + static constexpr std::size_t chunk_size = in_arity * xsimd::batch::size; + + /// Load an array of batches. + template + static XSIMD_INLINE auto load_batches(T const* ptr) -> batch_array + { + batch_array x; + for (std::size_t i = 0; i < x.size(); ++i) + { + x[i] = load_batch(ptr + i * xsimd::batch::size); + } + return x; + } + + /// Store an array of batches. + template + static XSIMD_INLINE void store_batches(batch_array const& x, T* ptr) + { + for (std::size_t i = 0; i < x.size(); ++i) + { + store_batch(x[i], ptr + i * xsimd::batch::size); + } + } + }; + /// Map fewer elements than a full step through a scratch buffer. template < typename Arch = xsimd::default_arch, typename T, typename U, typename Func> XSIMD_INLINE void map_unary_batch( T const* XSIMD_RESTRICT begin, - T const* XSIMD_RESTRICT end, U* XSIMD_RESTRICT out, + std::size_t count, Func&& func) { - constexpr std::size_t in_arity = batch_arity; - constexpr std::size_t out_arity = batch_arity; - constexpr std::size_t step = in_arity * xsimd::batch::size; - static_assert(step == out_arity * xsimd::batch::size); - auto mapper = internal::wrap_params_as_1d_arrays(std::forward(func)); - - assert(begin <= end); - assert(static_cast(end - begin) <= step); + using H = map_helper; - if (begin == end) [[unlikely]] + assert(count <= H::chunk_size); + if (count == 0) [[unlikely]] { return; } - alignas(Arch::alignment()) std::array input_buffer {}; - alignas(Arch::alignment()) std::array output_buffer; + auto mapper = internal::wrap_params_as_1d_arrays(std::forward(func)); + + alignas(Arch::alignment()) std::array input_buffer {}; + alignas(Arch::alignment()) std::array output_buffer; - const std::size_t count = static_cast(end - begin); std::memcpy(input_buffer.data(), begin, count * sizeof(T)); - store_batches( - mapper(load_batches(input_buffer.data())), - output_buffer.data()); + auto x = H::template load_batches(input_buffer.data()); + H::template store_batches(mapper(x), output_buffer.data()); std::memcpy(out, output_buffer.data(), count * sizeof(U)); } @@ -216,18 +228,13 @@ namespace xsimd::builder /// std::array, batch_arity>, both spanning the same element count. /// When arity is one, a callback over plain batches is accepted as well. template < - alignment align = alignment{}, - unary_options opts = unary_options{}, + alignment align = alignment {}, + unary_options opts = unary_options {}, typename Arch = xsimd::default_arch, typename T, typename U, typename Func> XSIMD_INLINE void map_unary(std::span in, std::span out, Func&& func) { - constexpr std::size_t in_arity = batch_arity; - constexpr std::size_t out_arity = batch_arity; - // Elements consumed and produced by a single call to func. - constexpr std::size_t step = in_arity * xsimd::batch::size; - static_assert(step == out_arity * xsimd::batch::size); - auto mapper = internal::wrap_params_as_1d_arrays(std::forward(func)); + using H = map_helper; // Input and output may not have the same alignment so it may be impossible // to get both aligned. We align preferably the output (more expensive @@ -239,77 +246,78 @@ namespace xsimd::builder assert(in.size() == out.size()); assert(!are_aliased(in, out)); + auto mapper = internal::wrap_params_as_1d_arrays(std::forward(func)); + if (in.empty()) [[unlikely]] { return; } - auto ot = out.data(); - auto it = in.data(); - auto const iend = in.data() + in.size(); + auto out_iter = out.data(); + auto in_iter = in.data(); + auto const in_end = in.data() + in.size(); if constexpr (!align.start_aligned) { // The span may be too short to reach the next alignment boundary. const auto head = std::min( - align_output ? bytes_to_next_aligned(ot, Arch::alignment()) / sizeof(U) - : bytes_to_next_aligned(it, Arch::alignment()) / sizeof(T), + align_output ? bytes_to_next_aligned(out_iter, Arch::alignment()) / sizeof(U) + : bytes_to_next_aligned(in_iter, Arch::alignment()) / sizeof(T), in.size()); - if (opts.pure && (head != 0) && (in.size() >= step)) + if (opts.pure && (head != 0) && (in.size() >= H::chunk_size)) { // Recompute the head as a full step, the body overwrites the excess. - store_batches( - mapper(load_batches(it)), - ot); + auto x = H::template load_batches(in_iter); + H::template store_batches(mapper(x), out_iter); } else { - map_unary_batch(it, it + head, ot, func); + map_unary_batch(in_iter, out_iter, head, func); } - it += head; - ot += head; + in_iter += head; + out_iter += head; } // Unrolled loop processing multiple steps at a time - while (static_cast(iend - it) >= opts.unroll_factor * step) + while (static_cast(in_end - in_iter) >= opts.unroll_factor * H::chunk_size) { - std::array, in_arity>, opts.unroll_factor> x; + std::array, opts.unroll_factor> x; for (std::size_t u = 0; u < opts.unroll_factor; ++u) { - x[u] = load_batches(it + u * step); + x[u] = H::template load_batches(in_iter + u * H::chunk_size); } for (std::size_t u = 0; u < opts.unroll_factor; ++u) { - store_batches(mapper(x[u]), ot + u * step); + H::template store_batches(mapper(x[u]), out_iter + u * H::chunk_size); } - it += opts.unroll_factor * step; - ot += opts.unroll_factor * step; + in_iter += opts.unroll_factor * H::chunk_size; + out_iter += opts.unroll_factor * H::chunk_size; } - while (static_cast(iend - it) >= step) + while (static_cast(in_end - in_iter) >= H::chunk_size) { - auto x = load_batches(it); - store_batches(mapper(x), ot); - it += step; - ot += step; + auto x = H::template load_batches(in_iter); + H::template store_batches(mapper(x), out_iter); + in_iter += H::chunk_size; + out_iter += H::chunk_size; } // Unlikely to be skipped, meant for users that know they allocate // a multiple of the batch size, such as in a local buffer if constexpr (!align.end_aligned) { - auto const oend = out.data() + out.size(); - if (opts.pure && (it != iend) && (in.size() >= step)) [[likely]] + auto const out_end = out.data() + out.size(); + if (opts.pure && (in_iter != in_end) && (in.size() >= H::chunk_size)) [[likely]] { // Recompute overlapping data, this time starting from the end. - auto x = load_batches(iend - step); - store_batches(mapper(x), oend - step); + auto x = H::template load_batches(in_end - H::chunk_size); + H::template store_batches(mapper(x), out_end - H::chunk_size); } else { - map_unary_batch(it, iend, ot, func); + map_unary_batch(in_iter, out_iter, in_end - in_iter, func); } } } From 6c65a1d0d4abdca878618b69bc6deebc961946b6 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Wed, 16 Sep 2026 16:24:57 +0200 Subject: [PATCH 19/27] Generic map_chunk --- include/xsimd_algorithm/builder.hpp | 72 ++++++++++++++--------------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/include/xsimd_algorithm/builder.hpp b/include/xsimd_algorithm/builder.hpp index 8eb00d4..534937a 100644 --- a/include/xsimd_algorithm/builder.hpp +++ b/include/xsimd_algorithm/builder.hpp @@ -146,12 +146,15 @@ namespace xsimd::builder } } - template + template + struct alignas(A::alignment()) alignas(T) aligned_array : std::array + { + }; + + template struct map_helper { - using out_t = Out; - using in_t = In; - static constexpr std::size_t min_elem_size = std::min(sizeof(out_t), sizeof(in_t)); + static constexpr std::size_t min_elem_size = std::min({ sizeof(Out), sizeof(In)... }); /// Number of batches of T spanning as many elements as one batch of the widest element. /// @@ -166,13 +169,11 @@ namespace xsimd::builder template using batch_array = std::array, batch_arity()>; - static constexpr std::size_t out_arity = batch_arity(); - static constexpr std::size_t in_arity = batch_arity(); - static constexpr std::size_t chunk_size = in_arity * xsimd::batch::size; + static constexpr std::size_t chunk_size = batch_arity() * xsimd::batch::size; /// Load an array of batches. template - static XSIMD_INLINE auto load_batches(T const* ptr) -> batch_array + XSIMD_INLINE static auto load_batches(T const* ptr) -> batch_array { batch_array x; for (std::size_t i = 0; i < x.size(); ++i) @@ -184,43 +185,40 @@ namespace xsimd::builder /// Store an array of batches. template - static XSIMD_INLINE void store_batches(batch_array const& x, T* ptr) + XSIMD_INLINE static void store_batches(batch_array const& x, T* ptr) { for (std::size_t i = 0; i < x.size(); ++i) { store_batch(x[i], ptr + i * xsimd::batch::size); } } - }; - /// Map fewer elements than a full step through a scratch buffer. - template < - typename Arch = xsimd::default_arch, - typename T, typename U, typename Func> - XSIMD_INLINE void map_unary_batch( - T const* XSIMD_RESTRICT begin, - U* XSIMD_RESTRICT out, - std::size_t count, - Func&& func) - { - using H = map_helper; - - assert(count <= H::chunk_size); - if (count == 0) [[unlikely]] + /// Map fewer elements than a full chunk through a scratch buffer. + template + XSIMD_INLINE static void map_chunk( + In const* XSIMD_RESTRICT... begin, + Out* XSIMD_RESTRICT out, + std::size_t count, + Func&& func) { - return; - } - - auto mapper = internal::wrap_params_as_1d_arrays(std::forward(func)); + assert(count <= chunk_size); + if (count == 0) [[unlikely]] + { + return; + } - alignas(Arch::alignment()) std::array input_buffer {}; - alignas(Arch::alignment()) std::array output_buffer; + constexpr auto read = [](T const* in, std::size_t cnt) + { + aligned_array in_buffer = {}; + std::memcpy(in_buffer.data(), in, cnt * sizeof(T)); + return load_batches(in_buffer.data()); + }; - std::memcpy(input_buffer.data(), begin, count * sizeof(T)); - auto x = H::template load_batches(input_buffer.data()); - H::template store_batches(mapper(x), output_buffer.data()); - std::memcpy(out, output_buffer.data(), count * sizeof(U)); - } + aligned_array out_buffer; + store_batches(func(read(begin, count)...), out_buffer.data()); + std::memcpy(out, out_buffer.data(), count * sizeof(Out)); + } + }; /// Apply func elementwise over in, writing as many elements to out. /// @@ -273,7 +271,7 @@ namespace xsimd::builder } else { - map_unary_batch(in_iter, out_iter, head, func); + H::map_chunk(in_iter, out_iter, head, mapper); } in_iter += head; out_iter += head; @@ -317,7 +315,7 @@ namespace xsimd::builder } else { - map_unary_batch(in_iter, out_iter, in_end - in_iter, func); + H::map_chunk(in_iter, out_iter, in_end - in_iter, mapper); } } } From f6816a8c342723b1c09f5566d1e292a38b9d31f3 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Wed, 16 Sep 2026 17:26:20 +0200 Subject: [PATCH 20/27] Refactor map_unrolled --- include/xsimd_algorithm/builder.hpp | 75 +++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/include/xsimd_algorithm/builder.hpp b/include/xsimd_algorithm/builder.hpp index 534937a..e5a3add 100644 --- a/include/xsimd_algorithm/builder.hpp +++ b/include/xsimd_algorithm/builder.hpp @@ -193,9 +193,53 @@ namespace xsimd::builder } } + template + XSIMD_INLINE static void map_chunk( + In const* XSIMD_RESTRICT... begin, + Out* XSIMD_RESTRICT out, + Func&& func) + { + store_batches(func(load_batches(begin)...), out); + } + + template + XSIMD_INLINE static void map_chunk_unaligned( + In const* XSIMD_RESTRICT... begin, + Out* XSIMD_RESTRICT out, + Func&& func) + { + return map_chunk(begin..., out, std::forward(func)); + } + + template + XSIMD_INLINE static void map_unrolled( + In const* XSIMD_RESTRICT... begin, + Out* XSIMD_RESTRICT out, + Func&& func, + std::index_sequence) + { + static constexpr std::size_t factor = sizeof...(step); + + constexpr auto read = [](T const* in) + { + std::array, factor> x; + ((x[step] = load_batches(in + step * chunk_size)), ...); + return x; + }; + + constexpr auto map = [](auto* out, auto&& f, auto const&... x) + { + auto map_one = [&](std::size_t s) + { store_batches(f(x[s]...), out + s * chunk_size); }; + (map_one(step), ...); + }; + + map(out, std::forward(func), read(begin)...); + } + /// Map fewer elements than a full chunk through a scratch buffer. template - XSIMD_INLINE static void map_chunk( + XSIMD_INLINE static void map_chunk_partial( In const* XSIMD_RESTRICT... begin, Out* XSIMD_RESTRICT out, std::size_t count, @@ -209,12 +253,12 @@ namespace xsimd::builder constexpr auto read = [](T const* in, std::size_t cnt) { - aligned_array in_buffer = {}; + alignas(A::alignment()) std::array in_buffer = {}; std::memcpy(in_buffer.data(), in, cnt * sizeof(T)); return load_batches(in_buffer.data()); }; - aligned_array out_buffer; + alignas(A::alignment()) std::array out_buffer; store_batches(func(read(begin, count)...), out_buffer.data()); std::memcpy(out, out_buffer.data(), count * sizeof(Out)); } @@ -266,12 +310,11 @@ namespace xsimd::builder if (opts.pure && (head != 0) && (in.size() >= H::chunk_size)) { // Recompute the head as a full step, the body overwrites the excess. - auto x = H::template load_batches(in_iter); - H::template store_batches(mapper(x), out_iter); + H::template map_chunk_unaligned(in_iter, out_iter, mapper); } else { - H::map_chunk(in_iter, out_iter, head, mapper); + H::map_chunk_partial(in_iter, out_iter, head, mapper); } in_iter += head; out_iter += head; @@ -280,24 +323,15 @@ namespace xsimd::builder // Unrolled loop processing multiple steps at a time while (static_cast(in_end - in_iter) >= opts.unroll_factor * H::chunk_size) { - std::array, opts.unroll_factor> x; - for (std::size_t u = 0; u < opts.unroll_factor; ++u) - { - x[u] = H::template load_batches(in_iter + u * H::chunk_size); - } - for (std::size_t u = 0; u < opts.unroll_factor; ++u) - { - H::template store_batches(mapper(x[u]), out_iter + u * H::chunk_size); - } - + H::template map_unrolled( + in_iter, out_iter, mapper, std::make_index_sequence()); in_iter += opts.unroll_factor * H::chunk_size; out_iter += opts.unroll_factor * H::chunk_size; } while (static_cast(in_end - in_iter) >= H::chunk_size) { - auto x = H::template load_batches(in_iter); - H::template store_batches(mapper(x), out_iter); + H::template map_chunk(in_iter, out_iter, mapper); in_iter += H::chunk_size; out_iter += H::chunk_size; } @@ -310,12 +344,11 @@ namespace xsimd::builder if (opts.pure && (in_iter != in_end) && (in.size() >= H::chunk_size)) [[likely]] { // Recompute overlapping data, this time starting from the end. - auto x = H::template load_batches(in_end - H::chunk_size); - H::template store_batches(mapper(x), out_end - H::chunk_size); + H::template map_chunk_unaligned(in_end - H::chunk_size, out_end - H::chunk_size, mapper); } else { - H::map_chunk(in_iter, out_iter, in_end - in_iter, mapper); + H::map_chunk_partial(in_iter, out_iter, in_end - in_iter, mapper); } } } From 039fa1fd536ab32ac38f66894ded2f0c615d9edf Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 17 Sep 2026 10:14:12 +0200 Subject: [PATCH 21/27] Refactor map_loop --- include/xsimd_algorithm/builder.hpp | 52 ++++++++++++++++++----------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/include/xsimd_algorithm/builder.hpp b/include/xsimd_algorithm/builder.hpp index e5a3add..4d62c01 100644 --- a/include/xsimd_algorithm/builder.hpp +++ b/include/xsimd_algorithm/builder.hpp @@ -146,11 +146,6 @@ namespace xsimd::builder } } - template - struct alignas(A::alignment()) alignas(T) aligned_array : std::array - { - }; - template struct map_helper { @@ -237,6 +232,27 @@ namespace xsimd::builder map(out, std::forward(func), read(begin)...); } + template + XSIMD_INLINE static auto map_loop( + In const* XSIMD_RESTRICT... begin, + Out* XSIMD_RESTRICT out, + std::size_t count, + Func&& func) -> std::size_t + { + constexpr auto steps = std::make_index_sequence(); + constexpr std::size_t total_step_size = unroll_factor * chunk_size; + + std::size_t remaining = count; + while (remaining >= total_step_size) + { + map_unrolled(begin..., out, func, steps); + ((begin += total_step_size), ...); + out += total_step_size; + remaining -= total_step_size; + } + return count - remaining; + } + /// Map fewer elements than a full chunk through a scratch buffer. template XSIMD_INLINE static void map_chunk_partial( @@ -320,21 +336,17 @@ namespace xsimd::builder out_iter += head; } - // Unrolled loop processing multiple steps at a time - while (static_cast(in_end - in_iter) >= opts.unroll_factor * H::chunk_size) - { - H::template map_unrolled( - in_iter, out_iter, mapper, std::make_index_sequence()); - in_iter += opts.unroll_factor * H::chunk_size; - out_iter += opts.unroll_factor * H::chunk_size; - } - - while (static_cast(in_end - in_iter) >= H::chunk_size) - { - H::template map_chunk(in_iter, out_iter, mapper); - in_iter += H::chunk_size; - out_iter += H::chunk_size; - } + // Unrolled loop processing multiple chunks at a time. + auto processed = H::template map_loop( + in_iter, out_iter, static_cast(in_end - in_iter), mapper); + in_iter += processed; + out_iter += processed; + + // Regular simd loop one chunk at a time. + processed = H::template map_loop( + in_iter, out_iter, static_cast(in_end - in_iter), mapper); + in_iter += processed; + out_iter += processed; // Unlikely to be skipped, meant for users that know they allocate // a multiple of the batch size, such as in a local buffer From 09030a174e00e675689b5363a49e25a1011cb4a2 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 17 Sep 2026 13:43:33 +0200 Subject: [PATCH 22/27] Refactor map_n helper --- include/xsimd_algorithm/builder.hpp | 428 ++++++++++++++++------------ 1 file changed, 253 insertions(+), 175 deletions(-) diff --git a/include/xsimd_algorithm/builder.hpp b/include/xsimd_algorithm/builder.hpp index 4d62c01..1635c78 100644 --- a/include/xsimd_algorithm/builder.hpp +++ b/include/xsimd_algorithm/builder.hpp @@ -144,147 +144,289 @@ namespace xsimd::builder } }; } - } - - template - struct map_helper - { - static constexpr std::size_t min_elem_size = std::min({ sizeof(Out), sizeof(In)... }); - /// Number of batches of T spanning as many elements as one batch of the widest element. - /// - /// Pairing that many batches on all side lets both sides advance by the same number of - /// elements, so a mapping stays elementwise regardless of the respective lane counts. - template - static constexpr std::size_t batch_arity() + template + struct map_helper { - return sizeof(T) / min_elem_size; - } + static constexpr std::size_t n_input = sizeof...(In); + static constexpr auto input_elem_size = std::array { sizeof(In)... }; - template - using batch_array = std::array, batch_arity()>; + static constexpr std::size_t min_elem_size = std::min({ sizeof(Out), sizeof(In)... }); + static constexpr std::size_t max_elem_size = std::max({ sizeof(Out), sizeof(In)... }); - static constexpr std::size_t chunk_size = batch_arity() * xsimd::batch::size; + /// Inputs and output may not have the same alignment so it may be impossible + /// to get all aligned. We align preferably the output (more expensive unaligned + /// stores) or otherwise one of the input. + static constexpr bool align_output = sizeof(Out) == max_elem_size; - /// Load an array of batches. - template - XSIMD_INLINE static auto load_batches(T const* ptr) -> batch_array - { - batch_array x; - for (std::size_t i = 0; i < x.size(); ++i) + static constexpr std::array get_align_inputs() { - x[i] = load_batch(ptr + i * xsimd::batch::size); + std::array out {}; + bool found = false; + for (std::size_t k = 0; k < out.size(); ++k) + { + if (found || align_output) + { + out[k] = false; + } + else + { + found = input_elem_size[k] == max_elem_size; + out[k] = found; + } + } + return out; } - return x; - } - /// Store an array of batches. - template - XSIMD_INLINE static void store_batches(batch_array const& x, T* ptr) - { - for (std::size_t i = 0; i < x.size(); ++i) + static constexpr std::array align_inputs = get_align_inputs(); + + static constexpr bool output_is_aligned = align_output || align.start_aligned; + + static constexpr std::array get_input_is_aligned() { - store_batch(x[i], ptr + i * xsimd::batch::size); + std::array out = align_inputs; + for (bool& b : out) + { + b = b || align.start_aligned; + } + return out; } - } - template - XSIMD_INLINE static void map_chunk( - In const* XSIMD_RESTRICT... begin, - Out* XSIMD_RESTRICT out, - Func&& func) - { - store_batches(func(load_batches(begin)...), out); - } + static constexpr std::array input_is_aligned = get_input_is_aligned(); - template - XSIMD_INLINE static void map_chunk_unaligned( - In const* XSIMD_RESTRICT... begin, - Out* XSIMD_RESTRICT out, - Func&& func) - { - return map_chunk(begin..., out, std::forward(func)); - } + /// Number of batches of T spanning as many elements as one batch of the widest element. + /// + /// Pairing that many batches on all side lets both sides advance by the same number of + /// elements, so a mapping stays elementwise regardless of the respective lane counts. + template + static constexpr std::size_t batch_arity() + { + return sizeof(T) / min_elem_size; + } - template - XSIMD_INLINE static void map_unrolled( - In const* XSIMD_RESTRICT... begin, - Out* XSIMD_RESTRICT out, - Func&& func, - std::index_sequence) - { - static constexpr std::size_t factor = sizeof...(step); + template + using batch_array = std::array, batch_arity()>; - constexpr auto read = [](T const* in) + static constexpr std::size_t chunk_size = batch_arity() * xsimd::batch::size; + + /// Load an array of batches. + template + XSIMD_INLINE static auto load_batches(T const* ptr) -> batch_array { - std::array, factor> x; - ((x[step] = load_batches(in + step * chunk_size)), ...); + batch_array x; + for (std::size_t i = 0; i < x.size(); ++i) + { + x[i] = load_batch(ptr + i * xsimd::batch::size); + } return x; - }; + } - constexpr auto map = [](auto* out, auto&& f, auto const&... x) + /// Store an array of batches. + template + XSIMD_INLINE static void store_batches(batch_array const& x, T* ptr) { - auto map_one = [&](std::size_t s) - { store_batches(f(x[s]...), out + s * chunk_size); }; - (map_one(step), ...); - }; + for (std::size_t i = 0; i < x.size(); ++i) + { + store_batch(x[i], ptr + i * xsimd::batch::size); + } + } - map(out, std::forward(func), read(begin)...); - } + /// Map a single unaligned chunk. + template + XSIMD_INLINE static void map_chunk_unaligned( + In const* XSIMD_RESTRICT... in, + Out* XSIMD_RESTRICT out, + Func&& func) + { + store_batches(func(load_batches(in)...), out); + } - template - XSIMD_INLINE static auto map_loop( - In const* XSIMD_RESTRICT... begin, - Out* XSIMD_RESTRICT out, - std::size_t count, - Func&& func) -> std::size_t - { - constexpr auto steps = std::make_index_sequence(); - constexpr std::size_t total_step_size = unroll_factor * chunk_size; + /// Map multiple chunks with a compile-time unrolled loop. + template < + std::array load_aligned, + bool store_aligned, + typename Func, + std::size_t... step> + XSIMD_INLINE static void map_unrolled( + In const* XSIMD_RESTRICT... in, + Out* XSIMD_RESTRICT out, + Func&& func, + std::index_sequence) + { + static constexpr std::size_t factor = sizeof...(step); + + constexpr auto read = [](T const* ptr, std::index_sequence) + { + std::array, factor> x; + auto load_one = [&](std::size_t s) + { + // Expand the parameter pack a for each alignment. + ((x[s] = load_batches(ptr + s * chunk_size)), ...); + }; + // Expand the step parameter pack: repeat the unrolled operation. + (load_one(step), ...); + return x; + }; + + constexpr auto map = [](auto* out, auto&& f, auto const&... x) + { + auto map_one = [&](std::size_t s) + { + // Expand the parameter pack a for each input. + store_batches(f(x[s]...), out + s * chunk_size); + }; + // Expand the step parameter pack: repeat the unrolled operation. + (map_one(step), ...); + }; + + map(out, std::forward(func), read(in, std::index_sequence_for {})...); + } - std::size_t remaining = count; - while (remaining >= total_step_size) + /// Map chunks in a loop with given unrolling factor. + /// + /// Return number of elements mapped. + template < + std::array load_aligned, + bool store_aligned, + std::size_t unroll_factor, + typename Func> + XSIMD_INLINE static auto map_loop( + In const* XSIMD_RESTRICT... in, + Out* XSIMD_RESTRICT out, + std::size_t count, + Func&& func) -> std::size_t { - map_unrolled(begin..., out, func, steps); - ((begin += total_step_size), ...); - out += total_step_size; - remaining -= total_step_size; + constexpr auto steps = std::make_index_sequence(); + constexpr std::size_t total_step_size = unroll_factor * chunk_size; + + std::size_t remaining = count; + while (remaining >= total_step_size) + { + map_unrolled(in..., out, func, steps); + ((in += total_step_size), ...); + out += total_step_size; + remaining -= total_step_size; + } + return count - remaining; } - return count - remaining; - } - /// Map fewer elements than a full chunk through a scratch buffer. - template - XSIMD_INLINE static void map_chunk_partial( - In const* XSIMD_RESTRICT... begin, - Out* XSIMD_RESTRICT out, - std::size_t count, - Func&& func) - { - assert(count <= chunk_size); - if (count == 0) [[unlikely]] + /// Map fewer elements than a full chunk through a scratch buffer. + template + XSIMD_INLINE static void map_chunk_partial( + In const* XSIMD_RESTRICT... begin, + Out* XSIMD_RESTRICT out, + std::size_t count, + Func&& func) { - return; + assert(count <= chunk_size); + if (count == 0) [[unlikely]] + { + return; + } + + constexpr auto read = [](T const* in, std::size_t cnt) + { + alignas(A::alignment()) std::array in_buffer = {}; + std::memcpy(in_buffer.data(), in, cnt * sizeof(T)); + return load_batches(in_buffer.data()); + }; + + alignas(A::alignment()) std::array out_buffer; + store_batches(func(read(begin, count)...), out_buffer.data()); + std::memcpy(out, out_buffer.data(), count * sizeof(Out)); } - constexpr auto read = [](T const* in, std::size_t cnt) + /// Given some pointers, return the number of element to process until desired alignment. + /// + /// The desired alignment is given via the compile-time parameters. + /// Only one can be true. + template align_in, bool align_out> + XSIMD_INLINE static auto elems_to_alignment(In const*... in, Out* out) -> std::size_t { - alignas(A::alignment()) std::array in_buffer = {}; - std::memcpy(in_buffer.data(), in, cnt * sizeof(T)); - return load_batches(in_buffer.data()); - }; + if constexpr (align_out) + { + return bytes_to_next_aligned(out, A::alignment()) / sizeof(Out); + } + else + { + constexpr auto iter = std::find(align_in.begin(), align_in.end(), true); + static_assert(iter < align_in.end()); + constexpr auto idx = iter - align_in.begin(); + auto to_align = std::array { in... }[idx]; + return bytes_to_next_aligned(to_align, A::alignment()) / input_elem_size[idx]; + } + } - alignas(A::alignment()) std::array out_buffer; - store_batches(func(read(begin, count)...), out_buffer.data()); - std::memcpy(out, out_buffer.data(), count * sizeof(Out)); - } - }; + template + XSIMD_INLINE static void map_n( + In const* XSIMD_RESTRICT... in, + Out* XSIMD_RESTRICT out, + std::size_t count, + Func&& func) + { + const auto advance = [&](std::size_t n) + { + ((in += n), ...); + out += n; + count -= n; + }; + + if (count == 0) [[unlikely]] + { + return; + } + + if constexpr (!align.start_aligned) + { + // The span may be too short to reach the next alignment boundary. + const auto to_alignment = elems_to_alignment(in..., out); + const auto head = std::min(to_alignment, count); + + if (opts.pure && (head != 0) && (count >= chunk_size)) + { + // Recompute the head as a full step, the body overwrites the excess. + map_chunk_unaligned(in..., out, func); + } + else + { + map_chunk_partial(in..., out, head, func); + } + advance(head); + } + + // Unrolled loop processing multiple chunks at a time. + auto processed = map_loop( + in..., out, count, func); + advance(processed); + + // Regular simd loop one chunk at a time. + processed = map_loop(in..., out, count, func); + advance(processed); + + // Unlikely to be skipped, meant for users that know they allocate + // a multiple of the batch size, such as in a local buffer + if constexpr (!align.end_aligned) + { + if (opts.pure && (count != 0) && (count >= chunk_size)) [[likely]] + { + // Recompute overlapping data, this time starting from the end. + map_chunk_unaligned((in + count - chunk_size)..., out + count - chunk_size, func); + } + else + { + map_chunk_partial(in..., out, count, func); + } + } + } + }; + } /// Apply func elementwise over in, writing as many elements to out. /// - /// Func maps a std::array, batch_arity> to a - /// std::array, batch_arity>, both spanning the same element count. - /// When arity is one, a callback over plain batches is accepted as well. + /// Func maps as many input as given to the function. + /// If input as of different sizes, then the larger ones must be passed as an array of + /// as many batches as the factor to the smallest element size, so that the function + /// processes a fixed amount of elements. template < alignment align = alignment {}, unary_options opts = unary_options {}, @@ -292,77 +434,13 @@ namespace xsimd::builder typename T, typename U, typename Func> XSIMD_INLINE void map_unary(std::span in, std::span out, Func&& func) { - using H = map_helper; - - // Input and output may not have the same alignment so it may be impossible - // to get both aligned. We align preferably the output (more expensive - // unaligned stores) or otherwise the input. - constexpr bool align_output = sizeof(U) >= sizeof(T); - constexpr bool load_is_aligned = align.start_aligned || !align_output; - constexpr bool store_is_aligned = align.start_aligned || align_output; + using H = internal::map_helper; assert(in.size() == out.size()); assert(!are_aliased(in, out)); auto mapper = internal::wrap_params_as_1d_arrays(std::forward(func)); - - if (in.empty()) [[unlikely]] - { - return; - } - - auto out_iter = out.data(); - auto in_iter = in.data(); - auto const in_end = in.data() + in.size(); - - if constexpr (!align.start_aligned) - { - // The span may be too short to reach the next alignment boundary. - const auto head = std::min( - align_output ? bytes_to_next_aligned(out_iter, Arch::alignment()) / sizeof(U) - : bytes_to_next_aligned(in_iter, Arch::alignment()) / sizeof(T), - in.size()); - - if (opts.pure && (head != 0) && (in.size() >= H::chunk_size)) - { - // Recompute the head as a full step, the body overwrites the excess. - H::template map_chunk_unaligned(in_iter, out_iter, mapper); - } - else - { - H::map_chunk_partial(in_iter, out_iter, head, mapper); - } - in_iter += head; - out_iter += head; - } - - // Unrolled loop processing multiple chunks at a time. - auto processed = H::template map_loop( - in_iter, out_iter, static_cast(in_end - in_iter), mapper); - in_iter += processed; - out_iter += processed; - - // Regular simd loop one chunk at a time. - processed = H::template map_loop( - in_iter, out_iter, static_cast(in_end - in_iter), mapper); - in_iter += processed; - out_iter += processed; - - // Unlikely to be skipped, meant for users that know they allocate - // a multiple of the batch size, such as in a local buffer - if constexpr (!align.end_aligned) - { - auto const out_end = out.data() + out.size(); - if (opts.pure && (in_iter != in_end) && (in.size() >= H::chunk_size)) [[likely]] - { - // Recompute overlapping data, this time starting from the end. - H::template map_chunk_unaligned(in_end - H::chunk_size, out_end - H::chunk_size, mapper); - } - else - { - H::map_chunk_partial(in_iter, out_iter, in_end - in_iter, mapper); - } - } + return H::template map_n(in.data(), out.data(), in.size(), mapper); } } From ee1498e9b42bffb1cd43ad554a79c07bc9d06e98 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 17 Sep 2026 14:12:07 +0200 Subject: [PATCH 23/27] Rename map options --- benchmark/bench_map_unary.cpp | 22 +++---- benchmark/bench_math.cpp | 4 +- benchmark/map_unary_utils.hpp | 8 +-- include/xsimd_algorithm/builder.hpp | 65 ++++++++++++------- include/xsimd_algorithm/math.hpp | 12 ++-- .../xsimd_test_utils/map_unary_data.hpp | 8 +-- .../include/xsimd_test_utils/math_ops.hpp | 4 +- test/test_builder.cpp | 2 +- test/test_math.cpp | 4 +- 9 files changed, 74 insertions(+), 55 deletions(-) diff --git a/benchmark/bench_map_unary.cpp b/benchmark/bench_map_unary.cpp index 9919da0..4145af4 100644 --- a/benchmark/bench_map_unary.cpp +++ b/benchmark/bench_map_unary.cpp @@ -21,8 +21,8 @@ namespace using xsimd::bench::bench_scalar; using xsimd::bench::bench_transform; using xsimd::bench::register_bench; - using xsimd::builder::alignment; - using xsimd::builder::unary_options; + using xsimd::builder::alignment_options; + using xsimd::builder::map_options; template void register_benches() @@ -32,8 +32,8 @@ namespace using aligned_alloc = typename xsimd::test::aligned_vector::allocator_type; using unaligned_alloc = typename xsimd::test::unaligned_vector::allocator_type; - constexpr auto noalign = alignment {}; - constexpr auto noopts = unary_options { .unroll_factor = 1, .pure = false }; + constexpr auto noalign = alignment_options {}; + constexpr auto noopts = map_options { .unroll_factor = 1, .pure = false }; register_bench("aligned/scalar", bench_scalar); register_bench("aligned/simd/transform", bench_transform); @@ -44,21 +44,21 @@ namespace "aligned/simd/map:pure", bench_map_unary< Op, aligned_alloc, arch, - noalign, unary_options { .unroll_factor = 1, .pure = true }>); + noalign, map_options { .unroll_factor = 1, .pure = true }>); register_bench( "aligned/simd/map:unroll4", bench_map_unary< - Op, aligned_alloc, arch, noalign, unary_options { .unroll_factor = 4 }>); + Op, aligned_alloc, arch, noalign, map_options { .unroll_factor = 4 }>); register_bench( "aligned/simd/map:noheader", bench_map_unary< Op, aligned_alloc, arch, - alignment { .start_aligned = true }, noopts>); + alignment_options { .start_aligned = true }, noopts>); register_bench( "aligned/simd/map:noheader+pure+unroll4", bench_map_unary< Op, aligned_alloc, arch, - alignment { .start_aligned = true }, unary_options { .unroll_factor = 4, .pure = true }>); + alignment_options { .start_aligned = true }, map_options { .unroll_factor = 4, .pure = true }>); register_bench("unaligned/scalar", bench_scalar); register_bench("unaligned/simd/transform", bench_transform); @@ -69,16 +69,16 @@ namespace "unaligned/simd/map:pure", bench_map_unary< Op, unaligned_alloc, arch, - noalign, unary_options { .unroll_factor = 1, .pure = true }>); + noalign, map_options { .unroll_factor = 1, .pure = true }>); register_bench( "unaligned/simd/map:unroll4", bench_map_unary< - Op, unaligned_alloc, arch, noalign, unary_options { .unroll_factor = 4 }>); + Op, unaligned_alloc, arch, noalign, map_options { .unroll_factor = 4 }>); register_bench( "unaligned/simd/map:pure+unroll4", bench_map_unary< Op, unaligned_alloc, arch, - noalign, unary_options { .unroll_factor = 4, .pure = true }>); + noalign, map_options { .unroll_factor = 4, .pure = true }>); } bool const registered = [] diff --git a/benchmark/bench_math.cpp b/benchmark/bench_math.cpp index 6b2e1dc..c6ba8cc 100644 --- a/benchmark/bench_math.cpp +++ b/benchmark/bench_math.cpp @@ -19,7 +19,7 @@ namespace using xsimd::bench::bench_map_unary; using xsimd::bench::bench_scalar; using xsimd::bench::register_bench; - using xsimd::builder::alignment; + using xsimd::builder::alignment_options; /// Register math benchmarks. /// @@ -42,7 +42,7 @@ namespace "hot/scalar", bench_scalar, /* sizes = */ { 1024 }); register_bench( "hot/simd", - bench_map_unary, + bench_map_unary, /* sizes = */ { 1024 }); } diff --git a/benchmark/map_unary_utils.hpp b/benchmark/map_unary_utils.hpp index d668863..ca46b6f 100644 --- a/benchmark/map_unary_utils.hpp +++ b/benchmark/map_unary_utils.hpp @@ -20,8 +20,8 @@ namespace xsimd::bench { - using xsimd::builder::alignment; - using xsimd::builder::unary_options; + using xsimd::builder::alignment_options; + using xsimd::builder::map_options; template void bench_unary(benchmark::State& state, Apply apply) @@ -47,8 +47,8 @@ namespace xsimd::bench typename Op, typename Alloc, typename Arch, - alignment aligned = alignment {}, - unary_options opts = unary_options {}> + alignment_options aligned = alignment_options {}, + map_options opts = map_options {}> void bench_map_unary(benchmark::State& state) { bench_unary( diff --git a/include/xsimd_algorithm/builder.hpp b/include/xsimd_algorithm/builder.hpp index 1635c78..d834efc 100644 --- a/include/xsimd_algorithm/builder.hpp +++ b/include/xsimd_algorithm/builder.hpp @@ -25,12 +25,6 @@ namespace xsimd::builder { - struct alignment - { - bool start_aligned = false; - bool end_aligned = false; - }; - /// Return the pointer before the input with the given alignment or itself if aligned. template XSIMD_INLINE auto prev_aligned(T* ptr, std::size_t alignment) -> T* @@ -58,8 +52,8 @@ namespace xsimd::builder } /// Check if two spans are aliasing each others (overlapping). - template - XSIMD_INLINE auto are_aliased(std::span lhs, std::span rhs) -> bool + template + XSIMD_INLINE auto are_aliased(std::span lhs, std::span rhs) -> bool { // Comparing pointers from unrelated objects is unspecified, integers are not. auto const lhs_begin = reinterpret_cast(lhs.data()); @@ -67,12 +61,6 @@ namespace xsimd::builder return (lhs_begin < rhs_begin + rhs.size_bytes()) && (rhs_begin < lhs_begin + lhs.size_bytes()); } - struct unary_options - { - std::size_t unroll_factor = 4; - bool pure = false; - }; - /// Load batch wrapper with an alignment as template parameter. template XSIMD_INLINE xsimd::batch load_batch(T const* ptr) @@ -101,6 +89,18 @@ namespace xsimd::builder } } + struct alignment_options + { + bool start_aligned = false; + bool end_aligned = false; + }; + + struct map_options + { + std::size_t unroll_factor = 4; + bool pure = false; + }; + namespace internal { template @@ -145,7 +145,7 @@ namespace xsimd::builder }; } - template + template struct map_helper { static constexpr std::size_t n_input = sizeof...(In); @@ -428,19 +428,38 @@ namespace xsimd::builder /// as many batches as the factor to the smallest element size, so that the function /// processes a fixed amount of elements. template < - alignment align = alignment {}, - unary_options opts = unary_options {}, + alignment_options align = alignment_options {}, + map_options opts = map_options {}, typename Arch = xsimd::default_arch, - typename T, typename U, typename Func> - XSIMD_INLINE void map_unary(std::span in, std::span out, Func&& func) + typename Func, + typename Out, + typename... In> + XSIMD_INLINE void map_n(Func&& func, Out&& out, In&&... in) { - using H = internal::map_helper; + using H = internal::map_helper< + opts, + align, + Arch, + typename std::remove_cvref_t::value_type, + typename std::remove_cvref_t::value_type...>; - assert(in.size() == out.size()); - assert(!are_aliased(in, out)); + assert((... && (in.size() == out.size()))); + assert((... && !are_aliased(std::span(in.data(), in.size()), std::span(out.data(), out.size())))); auto mapper = internal::wrap_params_as_1d_arrays(std::forward(func)); - return H::template map_n(in.data(), out.data(), in.size(), mapper); + return H::template map_n(in.data()..., out.data(), out.size(), mapper); + } + + template < + alignment_options align = alignment_options {}, + map_options opts = map_options {}, + typename Arch = xsimd::default_arch, + typename Func, + typename Out, + typename In> + XSIMD_INLINE void map_unary(In&& in, Out&& out, Func&& func) + { + return map_n(func, out, in); } } diff --git a/include/xsimd_algorithm/math.hpp b/include/xsimd_algorithm/math.hpp index 865cfa8..27574fe 100644 --- a/include/xsimd_algorithm/math.hpp +++ b/include/xsimd_algorithm/math.hpp @@ -16,36 +16,36 @@ namespace xsimd::algo { template < - xsimd::builder::alignment align = xsimd::builder::alignment{}, + xsimd::builder::alignment_options align = xsimd::builder::alignment_options{}, typename Arch = xsimd::default_arch, typename T> void sqrt(std::span in, std::span out) { - constexpr builder::unary_options opts = { .unroll_factor = 4, .pure = true }; + constexpr builder::map_options opts = { .unroll_factor = 4, .pure = true }; return xsimd::builder::map_unary( in, out, [](auto x) { return sqrt(x); }); } template < - xsimd::builder::alignment align = xsimd::builder::alignment{}, + xsimd::builder::alignment_options align = xsimd::builder::alignment_options{}, typename Arch = xsimd::default_arch, typename T> void abs(std::span in, std::span out) { - constexpr builder::unary_options opts = { .unroll_factor = 4, .pure = true }; + constexpr builder::map_options opts = { .unroll_factor = 4, .pure = true }; return xsimd::builder::map_unary( in, out, [](auto x) { return abs(x); }); } template < - xsimd::builder::alignment align = xsimd::builder::alignment{}, + xsimd::builder::alignment_options align = xsimd::builder::alignment_options{}, typename Arch = xsimd::default_arch, typename T> void exp(std::span in, std::span out) { - constexpr builder::unary_options opts = { .unroll_factor = 4, .pure = true }; + constexpr builder::map_options opts = { .unroll_factor = 4, .pure = true }; return xsimd::builder::map_unary( in, out, [](auto x) { return exp(x); }); diff --git a/test-utils/include/xsimd_test_utils/map_unary_data.hpp b/test-utils/include/xsimd_test_utils/map_unary_data.hpp index 8a1bdba..0fa959d 100644 --- a/test-utils/include/xsimd_test_utils/map_unary_data.hpp +++ b/test-utils/include/xsimd_test_utils/map_unary_data.hpp @@ -23,8 +23,8 @@ namespace xsimd::test { - using xsimd::builder::alignment; - using xsimd::builder::unary_options; + using xsimd::builder::alignment_options; + using xsimd::builder::map_options; /// Derives the scalar range application from the element-wise Derived::apply. template @@ -46,8 +46,8 @@ namespace xsimd::test } template < - alignment aligned = alignment {}, - unary_options opts = unary_options {}, + alignment_options aligned = alignment_options {}, + map_options opts = map_options {}, typename Arch = xsimd::default_arch> static void range_apply_map_unary(std::span in, std::span out) { diff --git a/test-utils/include/xsimd_test_utils/math_ops.hpp b/test-utils/include/xsimd_test_utils/math_ops.hpp index 114a4cf..9eeaf43 100644 --- a/test-utils/include/xsimd_test_utils/math_ops.hpp +++ b/test-utils/include/xsimd_test_utils/math_ops.hpp @@ -28,7 +28,7 @@ namespace xsimd::test static constexpr auto name = "sqrt"; - template + template static void apply(std::span in, std::span out) { xsimd::algo::sqrt(in, out); @@ -53,7 +53,7 @@ namespace xsimd::test static constexpr auto name = "abs"; - template + template static void apply(std::span in, std::span out) { xsimd::algo::abs(in, out); diff --git a/test/test_builder.cpp b/test/test_builder.cpp index 5088ebd..bbba5af 100644 --- a/test/test_builder.cpp +++ b/test/test_builder.cpp @@ -30,7 +30,7 @@ TEST_CASE("map_unary int32 to int64") const auto func = [](auto const& x) { return xsimd::widen(x + input_type { 1 }); }; - xsimd::builder::map_unary(xsimd::test::as_span(input), xsimd::test::as_span(output), func); + xsimd::builder::map_unary(input, output, func); for (std::size_t i = 0; i < size; ++i) { diff --git a/test/test_math.cpp b/test/test_math.cpp index 36caad7..2390594 100644 --- a/test/test_math.cpp +++ b/test/test_math.cpp @@ -16,7 +16,7 @@ namespace { - template + template void check_unary_math() { // Not a multiple of the batch size, to exercise the tail. @@ -58,7 +58,7 @@ TEST_CASE_TEMPLATE( SUBCASE("aligned without header") { - check_unary_math(); + check_unary_math(); } SUBCASE("aligned with header") From 022ea0d9e1263f50f98677c5f19370ff89e8bf48 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 17 Sep 2026 14:14:30 +0200 Subject: [PATCH 24/27] Rename builder.hpp > map.hpp --- benchmark/bench_map_unary.cpp | 6 +++--- benchmark/bench_math.cpp | 2 +- benchmark/map_unary_utils.hpp | 6 +++--- .../xsimd_algorithm/{builder.hpp => map.hpp} | 6 +++--- include/xsimd_algorithm/math.hpp | 20 +++++++++---------- .../xsimd_test_utils/map_unary_data.hpp | 8 ++++---- .../include/xsimd_test_utils/math_ops.hpp | 6 +++--- test/CMakeLists.txt | 2 +- test/{test_builder.cpp => test_map.cpp} | 6 +++--- test/test_math.cpp | 4 ++-- 10 files changed, 33 insertions(+), 33 deletions(-) rename include/xsimd_algorithm/{builder.hpp => map.hpp} (99%) rename test/{test_builder.cpp => test_map.cpp} (93%) diff --git a/benchmark/bench_map_unary.cpp b/benchmark/bench_map_unary.cpp index 4145af4..85193b9 100644 --- a/benchmark/bench_map_unary.cpp +++ b/benchmark/bench_map_unary.cpp @@ -11,7 +11,7 @@ #include #include "map_unary_utils.hpp" -#include "xsimd_algorithm/builder.hpp" +#include "xsimd_algorithm/map.hpp" #include "xsimd_test_utils/map_unary_data.hpp" #include "xsimd_test_utils/utils.hpp" @@ -21,8 +21,8 @@ namespace using xsimd::bench::bench_scalar; using xsimd::bench::bench_transform; using xsimd::bench::register_bench; - using xsimd::builder::alignment_options; - using xsimd::builder::map_options; + using xsimd::alignment_options; + using xsimd::map_options; template void register_benches() diff --git a/benchmark/bench_math.cpp b/benchmark/bench_math.cpp index c6ba8cc..eeabf47 100644 --- a/benchmark/bench_math.cpp +++ b/benchmark/bench_math.cpp @@ -19,7 +19,7 @@ namespace using xsimd::bench::bench_map_unary; using xsimd::bench::bench_scalar; using xsimd::bench::register_bench; - using xsimd::builder::alignment_options; + using xsimd::alignment_options; /// Register math benchmarks. /// diff --git a/benchmark/map_unary_utils.hpp b/benchmark/map_unary_utils.hpp index ca46b6f..68b54c4 100644 --- a/benchmark/map_unary_utils.hpp +++ b/benchmark/map_unary_utils.hpp @@ -15,13 +15,13 @@ #include #include "bench_utils.hpp" -#include "xsimd_algorithm/builder.hpp" +#include "xsimd_algorithm/map.hpp" #include "xsimd_test_utils/utils.hpp" namespace xsimd::bench { - using xsimd::builder::alignment_options; - using xsimd::builder::map_options; + using xsimd::alignment_options; + using xsimd::map_options; template void bench_unary(benchmark::State& state, Apply apply) diff --git a/include/xsimd_algorithm/builder.hpp b/include/xsimd_algorithm/map.hpp similarity index 99% rename from include/xsimd_algorithm/builder.hpp rename to include/xsimd_algorithm/map.hpp index d834efc..2b7d1cb 100644 --- a/include/xsimd_algorithm/builder.hpp +++ b/include/xsimd_algorithm/map.hpp @@ -6,8 +6,8 @@ * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ -#ifndef XSIMD_ALGORITHM_BUILDER_HPP -#define XSIMD_ALGORITHM_BUILDER_HPP +#ifndef XSIMD_ALGORITHM_MAP_HPP +#define XSIMD_ALGORITHM_MAP_HPP #include #include @@ -23,7 +23,7 @@ #include "./macros.hpp" -namespace xsimd::builder +namespace xsimd { /// Return the pointer before the input with the given alignment or itself if aligned. template diff --git a/include/xsimd_algorithm/math.hpp b/include/xsimd_algorithm/math.hpp index 27574fe..4b2a2f8 100644 --- a/include/xsimd_algorithm/math.hpp +++ b/include/xsimd_algorithm/math.hpp @@ -11,42 +11,42 @@ #include -#include "./builder.hpp" +#include "./map.hpp" namespace xsimd::algo { template < - xsimd::builder::alignment_options align = xsimd::builder::alignment_options{}, + xsimd::alignment_options align = xsimd::alignment_options{}, typename Arch = xsimd::default_arch, typename T> void sqrt(std::span in, std::span out) { - constexpr builder::map_options opts = { .unroll_factor = 4, .pure = true }; - return xsimd::builder::map_unary( + constexpr map_options opts = { .unroll_factor = 4, .pure = true }; + return xsimd::map_unary( in, out, [](auto x) { return sqrt(x); }); } template < - xsimd::builder::alignment_options align = xsimd::builder::alignment_options{}, + xsimd::alignment_options align = xsimd::alignment_options{}, typename Arch = xsimd::default_arch, typename T> void abs(std::span in, std::span out) { - constexpr builder::map_options opts = { .unroll_factor = 4, .pure = true }; - return xsimd::builder::map_unary( + constexpr map_options opts = { .unroll_factor = 4, .pure = true }; + return xsimd::map_unary( in, out, [](auto x) { return abs(x); }); } template < - xsimd::builder::alignment_options align = xsimd::builder::alignment_options{}, + xsimd::alignment_options align = xsimd::alignment_options{}, typename Arch = xsimd::default_arch, typename T> void exp(std::span in, std::span out) { - constexpr builder::map_options opts = { .unroll_factor = 4, .pure = true }; - return xsimd::builder::map_unary( + constexpr map_options opts = { .unroll_factor = 4, .pure = true }; + return xsimd::map_unary( in, out, [](auto x) { return exp(x); }); } diff --git a/test-utils/include/xsimd_test_utils/map_unary_data.hpp b/test-utils/include/xsimd_test_utils/map_unary_data.hpp index 0fa959d..e1e3fd9 100644 --- a/test-utils/include/xsimd_test_utils/map_unary_data.hpp +++ b/test-utils/include/xsimd_test_utils/map_unary_data.hpp @@ -16,15 +16,15 @@ #include #include -#include "xsimd_algorithm/builder.hpp" +#include "xsimd_algorithm/map.hpp" #include "xsimd_algorithm/stl/transform.hpp" #include "xsimd_test_utils/utils.hpp" namespace xsimd::test { - using xsimd::builder::alignment_options; - using xsimd::builder::map_options; + using xsimd::alignment_options; + using xsimd::map_options; /// Derives the scalar range application from the element-wise Derived::apply. template @@ -51,7 +51,7 @@ namespace xsimd::test typename Arch = xsimd::default_arch> static void range_apply_map_unary(std::span in, std::span out) { - return xsimd::builder::map_unary( + return xsimd::map_unary( in, out, [](auto x) { return Derived::apply_batch(x); }); } diff --git a/test-utils/include/xsimd_test_utils/math_ops.hpp b/test-utils/include/xsimd_test_utils/math_ops.hpp index 9eeaf43..4414e98 100644 --- a/test-utils/include/xsimd_test_utils/math_ops.hpp +++ b/test-utils/include/xsimd_test_utils/math_ops.hpp @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include "xsimd_test_utils/utils.hpp" @@ -28,7 +28,7 @@ namespace xsimd::test static constexpr auto name = "sqrt"; - template + template static void apply(std::span in, std::span out) { xsimd::algo::sqrt(in, out); @@ -53,7 +53,7 @@ namespace xsimd::test static constexpr auto name = "abs"; - template + template static void apply(std::span in, std::span out) { xsimd::algo::abs(in, out); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a6d7e87..7aba53b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -48,7 +48,7 @@ endif() set(XSIMD_ALGORITHM_TESTS main.cpp test_arange.cpp - test_builder.cpp + test_map.cpp test_iterator.cpp test_math.cpp test_reduce.cpp diff --git a/test/test_builder.cpp b/test/test_map.cpp similarity index 93% rename from test/test_builder.cpp rename to test/test_map.cpp index bbba5af..d3f7d05 100644 --- a/test/test_builder.cpp +++ b/test/test_map.cpp @@ -13,7 +13,7 @@ #include #include -#include "xsimd_algorithm/builder.hpp" +#include "xsimd_algorithm/map.hpp" /// Map unary test where one input batch pairs with two output batches. TEST_CASE("map_unary int32 to int64") @@ -30,7 +30,7 @@ TEST_CASE("map_unary int32 to int64") const auto func = [](auto const& x) { return xsimd::widen(x + input_type { 1 }); }; - xsimd::builder::map_unary(input, output, func); + xsimd::map_unary(input, output, func); for (std::size_t i = 0; i < size; ++i) { @@ -71,7 +71,7 @@ TEST_CASE("map_unary int64 to int32") xsimd::make_batch_constant()); }; - xsimd::builder::map_unary(xsimd::test::as_span(input), xsimd::test::as_span(output), func); + xsimd::map_unary(xsimd::test::as_span(input), xsimd::test::as_span(output), func); for (std::size_t i = 0; i < size; ++i) { diff --git a/test/test_math.cpp b/test/test_math.cpp index 2390594..b984dd9 100644 --- a/test/test_math.cpp +++ b/test/test_math.cpp @@ -16,7 +16,7 @@ namespace { - template + template void check_unary_math() { // Not a multiple of the batch size, to exercise the tail. @@ -58,7 +58,7 @@ TEST_CASE_TEMPLATE( SUBCASE("aligned without header") { - check_unary_math(); + check_unary_math(); } SUBCASE("aligned with header") From 24c0c7d5b9fc6aaeb03dc747649ca2d39c63151f Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 17 Sep 2026 14:44:37 +0200 Subject: [PATCH 25/27] Add binary benchmarks --- benchmark/CMakeLists.txt | 2 +- .../{bench_map_unary.cpp => bench_map.cpp} | 64 +++++++++ benchmark/bench_math.cpp | 24 +++- include/xsimd_algorithm/map.hpp | 13 ++ include/xsimd_algorithm/math.hpp | 24 ++++ .../xsimd_test_utils/map_binary_data.hpp | 122 ++++++++++++++++++ test/test_math.cpp | 53 ++++++++ 7 files changed, 300 insertions(+), 2 deletions(-) rename benchmark/{bench_map_unary.cpp => bench_map.cpp} (57%) create mode 100644 test-utils/include/xsimd_test_utils/map_binary_data.hpp diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index ef4e231..e214e36 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -19,7 +19,7 @@ find_package(benchmark REQUIRED) set(XSIMD_ALGORITHM_BENCHMARKS main.cpp bench_math.cpp - bench_map_unary.cpp + bench_map.cpp ) add_executable(benchmark_xsimd_algorithm ${XSIMD_ALGORITHM_BENCHMARKS}) diff --git a/benchmark/bench_map_unary.cpp b/benchmark/bench_map.cpp similarity index 57% rename from benchmark/bench_map_unary.cpp rename to benchmark/bench_map.cpp index 85193b9..562c53a 100644 --- a/benchmark/bench_map_unary.cpp +++ b/benchmark/bench_map.cpp @@ -7,19 +7,25 @@ ****************************************************************************/ #include +#include #include +#include "map_binary_utils.hpp" #include "map_unary_utils.hpp" #include "xsimd_algorithm/map.hpp" +#include "xsimd_test_utils/map_binary_data.hpp" #include "xsimd_test_utils/map_unary_data.hpp" #include "xsimd_test_utils/utils.hpp" namespace { + using xsimd::bench::bench_binary_scalar; + using xsimd::bench::bench_map_binary; using xsimd::bench::bench_map_unary; using xsimd::bench::bench_scalar; using xsimd::bench::bench_transform; + using xsimd::bench::register_binary_bench; using xsimd::bench::register_bench; using xsimd::alignment_options; using xsimd::map_options; @@ -81,6 +87,61 @@ namespace noalign, map_options { .unroll_factor = 4, .pure = true }>); } + template + void register_binary_benches() + { + using lhs_t = typename Op::lhs_t; + using arch = xsimd::default_arch; + using aligned_alloc = typename xsimd::test::aligned_vector::allocator_type; + using unaligned_alloc = typename xsimd::test::unaligned_vector::allocator_type; + + constexpr auto noalign = alignment_options {}; + constexpr auto noopts = map_options { .unroll_factor = 1, .pure = false }; + + register_binary_bench("aligned/scalar", bench_binary_scalar); + register_binary_bench( + "aligned/simd/map", + bench_map_binary); + register_binary_bench( + "aligned/simd/map:pure", + bench_map_binary< + Op, aligned_alloc, arch, + noalign, map_options { .unroll_factor = 1, .pure = true }>); + register_binary_bench( + "aligned/simd/map:unroll4", + bench_map_binary< + Op, aligned_alloc, arch, noalign, map_options { .unroll_factor = 4 }>); + register_binary_bench( + "aligned/simd/map:noheader", + bench_map_binary< + Op, aligned_alloc, arch, + alignment_options { .start_aligned = true }, noopts>); + register_binary_bench( + "aligned/simd/map:noheader+pure+unroll4", + bench_map_binary< + Op, aligned_alloc, arch, + alignment_options { .start_aligned = true }, map_options { .unroll_factor = 4, .pure = true }>); + + register_binary_bench("unaligned/scalar", bench_binary_scalar); + register_binary_bench( + "unaligned/simd/map", + bench_map_binary); + register_binary_bench( + "unaligned/simd/map:pure", + bench_map_binary< + Op, unaligned_alloc, arch, + noalign, map_options { .unroll_factor = 1, .pure = true }>); + register_binary_bench( + "unaligned/simd/map:unroll4", + bench_map_binary< + Op, unaligned_alloc, arch, noalign, map_options { .unroll_factor = 4 }>); + register_binary_bench( + "unaligned/simd/map:pure+unroll4", + bench_map_binary< + Op, unaligned_alloc, arch, + noalign, map_options { .unroll_factor = 4, .pure = true }>); + } + bool const registered = [] { register_benches>(); @@ -89,6 +150,9 @@ namespace register_benches>(); register_benches>(); register_benches>(); + register_binary_benches>(); + register_binary_benches>(); + register_binary_benches(); return true; }(); } diff --git a/benchmark/bench_math.cpp b/benchmark/bench_math.cpp index eeabf47..b5d4d21 100644 --- a/benchmark/bench_math.cpp +++ b/benchmark/bench_math.cpp @@ -10,14 +10,19 @@ #include +#include "map_binary_utils.hpp" #include "map_unary_utils.hpp" +#include "xsimd_test_utils/map_binary_data.hpp" #include "xsimd_test_utils/map_unary_data.hpp" #include "xsimd_test_utils/utils.hpp" namespace { + using xsimd::bench::bench_binary_scalar; + using xsimd::bench::bench_map_binary; using xsimd::bench::bench_map_unary; using xsimd::bench::bench_scalar; + using xsimd::bench::register_binary_bench; using xsimd::bench::register_bench; using xsimd::alignment_options; @@ -25,7 +30,7 @@ namespace /// /// To avoid an explosion of benchmarks, we only add a simple aligned benchmark. /// This will let us know the performance of the xsimd wrappers. - /// See bench_map_unary for benchmarks on the different flavor of mapping, alignment, + /// See bench_map for benchmarks on the different flavor of mapping, alignment, /// headers and trailers. /// /// This benchmark aims to test raw the performance of intrinsic, unrelated to @@ -46,6 +51,21 @@ namespace /* sizes = */ { 1024 }); } + template + void register_binary_benches() + { + using lhs_t = typename Op::lhs_t; + using arch = xsimd::default_arch; + using aligned_alloc = typename xsimd::test::aligned_vector::allocator_type; + + register_binary_bench( + "hot/scalar", bench_binary_scalar, /* sizes = */ { 1024 }); + register_binary_bench( + "hot/simd", + bench_map_binary, + /* sizes = */ { 1024 }); + } + bool const registered = [] { register_benches>(); @@ -57,6 +77,8 @@ namespace register_benches>(); register_benches>(); register_benches>(); + register_binary_benches>(); + register_binary_benches>(); return true; }(); } diff --git a/include/xsimd_algorithm/map.hpp b/include/xsimd_algorithm/map.hpp index 2b7d1cb..c1e65b7 100644 --- a/include/xsimd_algorithm/map.hpp +++ b/include/xsimd_algorithm/map.hpp @@ -461,6 +461,19 @@ namespace xsimd { return map_n(func, out, in); } + + template < + alignment_options align = alignment_options {}, + map_options opts = map_options {}, + typename Arch = xsimd::default_arch, + typename Func, + typename Out, + typename Lhs, + typename Rhs> + XSIMD_INLINE void map_binary(Lhs&& lhs, Rhs&& rhs, Out&& out, Func&& func) + { + return map_n(func, out, lhs, rhs); + } } #endif diff --git a/include/xsimd_algorithm/math.hpp b/include/xsimd_algorithm/math.hpp index 4b2a2f8..514b0c4 100644 --- a/include/xsimd_algorithm/math.hpp +++ b/include/xsimd_algorithm/math.hpp @@ -50,6 +50,30 @@ namespace xsimd::algo in, out, [](auto x) { return exp(x); }); } + + template < + xsimd::alignment_options align = xsimd::alignment_options{}, + typename Arch = xsimd::default_arch, + typename T> + void add(std::span lhs, std::span rhs, std::span out) + { + constexpr map_options opts = { .unroll_factor = 4, .pure = true }; + return xsimd::map_binary( + lhs, rhs, out, [](auto x, auto y) + { return x + y; }); + } + + template < + xsimd::alignment_options align = xsimd::alignment_options{}, + typename Arch = xsimd::default_arch, + typename T> + void multiply(std::span lhs, std::span rhs, std::span out) + { + constexpr map_options opts = { .unroll_factor = 4, .pure = true }; + return xsimd::map_binary( + lhs, rhs, out, [](auto x, auto y) + { return x * y; }); + } } #endif diff --git a/test-utils/include/xsimd_test_utils/map_binary_data.hpp b/test-utils/include/xsimd_test_utils/map_binary_data.hpp new file mode 100644 index 0000000..4df143e --- /dev/null +++ b/test-utils/include/xsimd_test_utils/map_binary_data.hpp @@ -0,0 +1,122 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ALGORITHM_TEST_UTILS_MAP_BINARY_DATA_HPP +#define XSIMD_ALGORITHM_TEST_UTILS_MAP_BINARY_DATA_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "xsimd_algorithm/map.hpp" + +#include "xsimd_test_utils/utils.hpp" + +namespace xsimd::test +{ + using xsimd::alignment_options; + using xsimd::map_options; + + /// Derives the scalar range application from the element-wise Derived::apply. + template + struct binary_op + { + using lhs_t = Lhs; + using rhs_t = Rhs; + using output_t = Out; + + template + using rebind_allocator = typename std::allocator_traits::template rebind_alloc; + + static void range_apply_scalar(std::span lhs, std::span rhs, std::span out) + { + for (std::size_t i = 0; i < lhs.size(); ++i) + { + out[i] = Derived::apply_scalar(lhs[i], rhs[i]); + } + } + + template < + alignment_options aligned = alignment_options {}, + map_options opts = map_options {}, + typename Arch = xsimd::default_arch> + static void range_apply_map_binary(std::span lhs, std::span rhs, std::span out) + { + return xsimd::map_binary( + lhs, rhs, out, [](auto x, auto y) + { return Derived::apply_batch(x, y); }); + } + + template + static auto make_input_output(std::size_t size) + -> std::tuple< + std::vector>, + std::vector>, + std::vector>> + { + auto lhs = make_arange>(size, -static_cast(size / 2)); + auto rhs = make_arange>(size, static_cast(3)); + auto output = std::vector>(size); + return { std::move(lhs), std::move(rhs), std::move(output) }; + } + }; + + /******************* + * Test fixtures * + *******************/ + + template + struct add_op : binary_op, T> + { + static constexpr auto name = "add"; + static constexpr bool pure = true; + + static auto apply_scalar(T x, T y) { return static_cast(x + y); } + + template + static auto apply_batch(xsimd::batch x, xsimd::batch y) { return x + y; } + }; + + template + struct multiply_op : binary_op, T> + { + static constexpr auto name = "multiply"; + static constexpr bool pure = true; + + static auto apply_scalar(T x, T y) { return static_cast(x * y); } + + template + static auto apply_batch(xsimd::batch x, xsimd::batch y) { return x * y; } + }; + + /// Multiply an int32 by a double, one int32 batch with two double batches. + struct mixed_multiply_op : binary_op + { + static constexpr auto name = "mixed_multiply"; + static constexpr bool pure = true; + + static auto apply_scalar(std::int32_t x, double y) { return static_cast(x) * y; } + + template + static auto apply_batch(xsimd::batch x, std::array, 2> const& y) + { + auto const wide = xsimd::widen(x); + return std::array { + xsimd::batch_cast(wide[0]) * y[0], + xsimd::batch_cast(wide[1]) * y[1], + }; + } + }; +} + +#endif diff --git a/test/test_math.cpp b/test/test_math.cpp index b984dd9..26c1cf9 100644 --- a/test/test_math.cpp +++ b/test/test_math.cpp @@ -11,6 +11,7 @@ #include +#include #include #include @@ -39,6 +40,31 @@ namespace } } } + + template + void check_binary_math() + { + // Not a multiple of the batch size, to exercise the tail. + constexpr std::size_t test_size = 94; + + auto [lhs, rhs, output] = Op::template make_input_output(test_size); + + Op::template range_apply_map_binary( + xsimd::test::as_span(lhs), xsimd::test::as_span(rhs), xsimd::test::as_span(output)); + + for (std::size_t i = 0; i < lhs.size(); ++i) + { + CAPTURE(i); + if constexpr (std::is_floating_point_v) + { + CHECK(output[i] == doctest::Approx(Op::apply_scalar(lhs[i], rhs[i]))); + } + else + { + CHECK(output[i] == Op::apply_scalar(lhs[i], rhs[i])); + } + } + } } TEST_CASE_TEMPLATE( @@ -71,3 +97,30 @@ TEST_CASE_TEMPLATE( check_unary_math(); } } + +TEST_CASE_TEMPLATE( + "binary math", + Op, + xsimd::test::add_op, + xsimd::test::multiply_op, + xsimd::test::mixed_multiply_op) +{ + using lhs_t = typename Op::lhs_t; + using aligned_allocator = typename xsimd::test::aligned_vector::allocator_type; + using unaligned_allocator = typename xsimd::test::unaligned_vector::allocator_type; + + SUBCASE("aligned without header") + { + check_binary_math(); + } + + SUBCASE("aligned with header") + { + check_binary_math(); + } + + SUBCASE("unaligned with header") + { + check_binary_math(); + } +} From 20784fc8703a9ed59c2f68b4c9e375a9801cada4 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 17 Sep 2026 15:06:45 +0200 Subject: [PATCH 26/27] Optimize small memcpy --- include/xsimd_algorithm/map.hpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/include/xsimd_algorithm/map.hpp b/include/xsimd_algorithm/map.hpp index c1e65b7..f705d6a 100644 --- a/include/xsimd_algorithm/map.hpp +++ b/include/xsimd_algorithm/map.hpp @@ -61,6 +61,26 @@ namespace xsimd return (lhs_begin < rhs_begin + rhs.size_bytes()) && (rhs_begin < lhs_begin + lhs.size_bytes()); } + /// Copy fewer than 2 * k elements without a call to memcpy. + /// + /// For count in [k, 2k), two overlapping copies of k elements cover the range; since k is a + /// compile-time constant, each memcpy compiles down to a few fixed-size loads and stores. + template + XSIMD_INLINE void copy_small(T* XSIMD_RESTRICT dst, T const* XSIMD_RESTRICT src, std::size_t count) + { + assert(std::has_single_bit(k)); + assert(count < 2 * k); + if (count >= k) + { + std::memcpy(dst, src, k * sizeof(T)); + std::memcpy(dst + count - k, src + count - k, k * sizeof(T)); + } + else if (k > 1) + { + copy_small(dst, src, count); + } + } + /// Load batch wrapper with an alignment as template parameter. template XSIMD_INLINE xsimd::batch load_batch(T const* ptr) @@ -327,13 +347,13 @@ namespace xsimd constexpr auto read = [](T const* in, std::size_t cnt) { alignas(A::alignment()) std::array in_buffer = {}; - std::memcpy(in_buffer.data(), in, cnt * sizeof(T)); + copy_small(in_buffer.data(), in, cnt); return load_batches(in_buffer.data()); }; alignas(A::alignment()) std::array out_buffer; store_batches(func(read(begin, count)...), out_buffer.data()); - std::memcpy(out, out_buffer.data(), count * sizeof(Out)); + copy_small(out, out_buffer.data(), count); } /// Given some pointers, return the number of element to process until desired alignment. From eadbce2b103b8748a8885d1acd00b96796fa204b Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 17 Sep 2026 15:27:31 +0200 Subject: [PATCH 27/27] Fix CI --- include/xsimd_algorithm/map.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xsimd_algorithm/map.hpp b/include/xsimd_algorithm/map.hpp index f705d6a..12e9ed9 100644 --- a/include/xsimd_algorithm/map.hpp +++ b/include/xsimd_algorithm/map.hpp @@ -467,7 +467,7 @@ namespace xsimd assert((... && !are_aliased(std::span(in.data(), in.size()), std::span(out.data(), out.size())))); auto mapper = internal::wrap_params_as_1d_arrays(std::forward(func)); - return H::template map_n(in.data()..., out.data(), out.size(), mapper); + return H::map_n(in.data()..., out.data(), out.size(), mapper); } template <