From bbeeeb2ba50d2e2e6af29c72ba5894dbd05e8c10 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 4 Aug 2026 15:44:59 +0200 Subject: [PATCH 1/2] Move add to v15 on arm --- include/xsimd/arch/utils/sve.hpp | 51 +++++++ include/xsimd/arch/xsimd_isa.hpp | 5 + include/xsimd/arch/xsimd_neon.hpp | 36 ----- include/xsimd/arch/xsimd_neon64.hpp | 13 +- include/xsimd/arch/xsimd_sve.hpp | 159 +++++++++----------- include/xsimd/types/xsimd_all_registers.hpp | 4 +- include/xsimd/types/xsimd_arm_registers.hpp | 16 ++ include/xsimd/types/xsimd_batch.hpp | 3 +- include/xsimd/types/xsimd_sve_register.hpp | 17 +++ include/xsimd/v15/arithmetic/arm.hpp | 91 +++++++++++ include/xsimd/v15/kernel_fwd.hpp | 20 +++ 11 files changed, 280 insertions(+), 135 deletions(-) create mode 100644 include/xsimd/arch/utils/sve.hpp create mode 100644 include/xsimd/types/xsimd_arm_registers.hpp create mode 100644 include/xsimd/v15/arithmetic/arm.hpp create mode 100644 include/xsimd/v15/kernel_fwd.hpp diff --git a/include/xsimd/arch/utils/sve.hpp b/include/xsimd/arch/utils/sve.hpp new file mode 100644 index 000000000..7176b3d97 --- /dev/null +++ b/include/xsimd/arch/utils/sve.hpp @@ -0,0 +1,51 @@ +/**************************************************************************** + * Copyright (c) xsimd contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ARCH_UTILS_SVE_HPP +#define XSIMD_ARCH_UTILS_SVE_HPP + +#include "../../config/xsimd_macros.hpp" +#include "../../types/xsimd_sve_register.hpp" + +#include + +// Define a inline namespace with the explicit SVE vector size to avoid ODR violation +// When dynamically dispatching between different SVE sizes. +// While most code is safe from ODR violation as the size is already encoded in the +// register (and hence batch) types, utilities can quickly fall prone to this issue. +#define XSIMD_SVE_NAMESPACE XSIMD_CONCAT(sve, XSIMD_SVE_BITS) + +namespace xsimd::kernel::detail +{ + inline namespace XSIMD_SVE_NAMESPACE + { + template + XSIMD_INLINE auto svptrue() noexcept + { +#if XSIMD_WITH_SVE + if constexpr (sizeof(T) == 1) + { + return svptrue_b8(); + } + else if constexpr (sizeof(T) == 2) + { + return svptrue_b16(); + } + else if constexpr (sizeof(T) == 4) + { + return svptrue_b32(); + } + else if constexpr (sizeof(T) == 8) + { + return svptrue_b64(); + } +#endif + } + } +} +#endif diff --git a/include/xsimd/arch/xsimd_isa.hpp b/include/xsimd/arch/xsimd_isa.hpp index 87aaa6c4a..061c73598 100644 --- a/include/xsimd/arch/xsimd_isa.hpp +++ b/include/xsimd/arch/xsimd_isa.hpp @@ -16,6 +16,11 @@ #include "../config/xsimd_arch.hpp" #include "./xsimd_common_fwd.hpp" +// v15 API under migration +#if XSIMD_WITH_NEON || XSIMD_WITH_SVE +#include "../v15/arithmetic/arm.hpp" +#endif + #if XSIMD_WITH_EMULATED #include "./xsimd_emulated.hpp" #endif diff --git a/include/xsimd/arch/xsimd_neon.hpp b/include/xsimd/arch/xsimd_neon.hpp index c0aff00bb..b22587ed2 100644 --- a/include/xsimd/arch/xsimd_neon.hpp +++ b/include/xsimd/arch/xsimd_neon.hpp @@ -811,42 +811,6 @@ namespace xsimd return vnegq_f32(rhs); } - /******* - * add * - *******/ - - namespace wrap - { - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template , int> = 0> - XSIMD_INLINE uint8x16_t x_vaddq(uint8x16_t a, uint8x16_t b) noexcept { return vaddq_u8(a, b); } - template , int> = 0> - XSIMD_INLINE int8x16_t x_vaddq(int8x16_t a, int8x16_t b) noexcept { return vaddq_s8(a, b); } - template , int> = 0> - XSIMD_INLINE uint16x8_t x_vaddq(uint16x8_t a, uint16x8_t b) noexcept { return vaddq_u16(a, b); } - template , int> = 0> - XSIMD_INLINE int16x8_t x_vaddq(int16x8_t a, int16x8_t b) noexcept { return vaddq_s16(a, b); } - template , int> = 0> - XSIMD_INLINE uint32x4_t x_vaddq(uint32x4_t a, uint32x4_t b) noexcept { return vaddq_u32(a, b); } - template , int> = 0> - XSIMD_INLINE int32x4_t x_vaddq(int32x4_t a, int32x4_t b) noexcept { return vaddq_s32(a, b); } - template , int> = 0> - XSIMD_INLINE uint64x2_t x_vaddq(uint64x2_t a, uint64x2_t b) noexcept { return vaddq_u64(a, b); } - template , int> = 0> - XSIMD_INLINE int64x2_t x_vaddq(int64x2_t a, int64x2_t b) noexcept { return vaddq_s64(a, b); } - template , int> = 0> - XSIMD_INLINE float32x4_t x_vaddq(float32x4_t a, float32x4_t b) noexcept { return vaddq_f32(a, b); } - } - - template = 0> - XSIMD_INLINE batch add(batch const& lhs, batch const& rhs, requires_arch) noexcept - { - using register_type = typename batch::register_type; - return wrap::x_vaddq>(register_type(lhs), register_type(rhs)); - } - /******* * avg * *******/ diff --git a/include/xsimd/arch/xsimd_neon64.hpp b/include/xsimd/arch/xsimd_neon64.hpp index 809ef8fa5..9903763aa 100644 --- a/include/xsimd/arch/xsimd_neon64.hpp +++ b/include/xsimd/arch/xsimd_neon64.hpp @@ -14,6 +14,7 @@ #include "../types/xsimd_neon64_register.hpp" #include "../types/xsimd_utils.hpp" +#include "../v15/kernel_fwd.hpp" #include "./xsimd_neon.hpp" #include @@ -352,16 +353,6 @@ namespace xsimd return vnegq_f64(rhs); } - /******* - * add * - *******/ - - template - XSIMD_INLINE batch add(batch const& lhs, batch const& rhs, requires_arch) noexcept - { - return vaddq_f64(lhs, rhs); - } - /******** * sadd * ********/ @@ -369,7 +360,7 @@ namespace xsimd template XSIMD_INLINE batch sadd(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return add(lhs, rhs, neon64 {}); + return ::xsimd::kernel::add(lhs, rhs); } /******* diff --git a/include/xsimd/arch/xsimd_sve.hpp b/include/xsimd/arch/xsimd_sve.hpp index d47525c8e..b657b5979 100644 --- a/include/xsimd/arch/xsimd_sve.hpp +++ b/include/xsimd/arch/xsimd_sve.hpp @@ -40,15 +40,6 @@ namespace xsimd using xsimd::index; using xsimd::types::detail::sve_vector_type; - // predicate creation - XSIMD_INLINE svbool_t ptrue_impl(index<1>) noexcept { return svptrue_b8(); } - XSIMD_INLINE svbool_t ptrue_impl(index<2>) noexcept { return svptrue_b16(); } - XSIMD_INLINE svbool_t ptrue_impl(index<4>) noexcept { return svptrue_b32(); } - XSIMD_INLINE svbool_t ptrue_impl(index<8>) noexcept { return svptrue_b64(); } - - template - XSIMD_INLINE svbool_t ptrue() noexcept { return ptrue_impl(index {}); } - // count active lanes in a predicate XSIMD_INLINE uint64_t pcount_impl(svbool_t p, index<1>) noexcept { return svcntp_b8(p, p); } XSIMD_INLINE uint64_t pcount_impl(svbool_t p, index<2>) noexcept { return svcntp_b16(p, p); } @@ -81,7 +72,7 @@ namespace xsimd template = 0> XSIMD_INLINE batch load_aligned(T const* src, convert, requires_arch) noexcept { - return svld1(detail_sve::ptrue(), reinterpret_cast const*>(src)); + return svld1(detail::svptrue(), reinterpret_cast const*>(src)); } template = 0> @@ -114,7 +105,7 @@ namespace xsimd XSIMD_INLINE batch, A> load_complex_aligned(std::complex const* mem, convert>, requires_arch) noexcept { const T* buf = reinterpret_cast(mem); - const auto tmp = svld2(detail_sve::ptrue(), buf); + const auto tmp = svld2(detail::svptrue(), buf); const auto real = svget2(tmp, 0); const auto imag = svget2(tmp, 1); return batch, A> { real, imag }; @@ -133,7 +124,7 @@ namespace xsimd template = 0> XSIMD_INLINE void store_aligned(T* dst, batch const& src, requires_arch) noexcept { - svst1(detail_sve::ptrue(), reinterpret_cast*>(dst), src); + svst1(detail::svptrue(), reinterpret_cast*>(dst), src); } template = 0> @@ -169,7 +160,7 @@ namespace xsimd tmp = svset2(tmp, 0, src.real()); tmp = svset2(tmp, 1, src.imag()); T* buf = reinterpret_cast(dst); - svst2(detail_sve::ptrue(), buf, tmp); + svst2(detail::svptrue(), buf, tmp); } template = 0> @@ -192,14 +183,14 @@ namespace xsimd template = 0> XSIMD_INLINE void scatter(batch const& src, T* dst, batch const& index, kernel::requires_arch) noexcept { - svst1_scatter_index(detail_sve::ptrue(), dst, index.data, src.data); + svst1_scatter_index(detail::svptrue(), dst, index.data, src.data); } // gather template = 0> XSIMD_INLINE batch gather(batch const&, T const* src, batch const& index, kernel::requires_arch) noexcept { - return svld1_gather_index(detail_sve::ptrue(), src, index.data); + return svld1_gather_index(detail::svptrue(), src, index.data); } /******************** @@ -281,7 +272,7 @@ namespace xsimd template = 0> XSIMD_INLINE batch add(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svadd_x(detail_sve::ptrue(), lhs, rhs); + return svadd_x(detail::svptrue(), lhs, rhs); } // sadd @@ -295,7 +286,7 @@ namespace xsimd template = 0> XSIMD_INLINE batch sub(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svsub_x(detail_sve::ptrue(), lhs, rhs); + return svsub_x(detail::svptrue(), lhs, rhs); } // ssub @@ -309,66 +300,66 @@ namespace xsimd template = 0> XSIMD_INLINE batch mul(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svmul_x(detail_sve::ptrue(), lhs, rhs); + return svmul_x(detail::svptrue(), lhs, rhs); } // mul_hi template , int>::type = 0> XSIMD_INLINE batch mul_hi(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svmulh_x(detail_sve::ptrue(), lhs, rhs); + return svmulh_x(detail::svptrue(), lhs, rhs); } // div template = 4, int> = 0> XSIMD_INLINE batch div(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svdiv_x(detail_sve::ptrue(), lhs, rhs); + return svdiv_x(detail::svptrue(), lhs, rhs); } // max template = 0> XSIMD_INLINE batch max(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svmax_x(detail_sve::ptrue(), lhs, rhs); + return svmax_x(detail::svptrue(), lhs, rhs); } // min template = 0> XSIMD_INLINE batch min(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svmin_x(detail_sve::ptrue(), lhs, rhs); + return svmin_x(detail::svptrue(), lhs, rhs); } // neg template = 0> XSIMD_INLINE batch neg(batch const& arg, requires_arch) noexcept { - return svreinterpret_u8(svneg_x(detail_sve::ptrue(), svreinterpret_s8(static_cast>(arg)))); + return svreinterpret_u8(svneg_x(detail::svptrue(), svreinterpret_s8(static_cast>(arg)))); } template = 0> XSIMD_INLINE batch neg(batch const& arg, requires_arch) noexcept { - return svreinterpret_u16(svneg_x(detail_sve::ptrue(), svreinterpret_s16(static_cast>(arg)))); + return svreinterpret_u16(svneg_x(detail::svptrue(), svreinterpret_s16(static_cast>(arg)))); } template = 0> XSIMD_INLINE batch neg(batch const& arg, requires_arch) noexcept { - return svreinterpret_u32(svneg_x(detail_sve::ptrue(), svreinterpret_s32(static_cast>(arg)))); + return svreinterpret_u32(svneg_x(detail::svptrue(), svreinterpret_s32(static_cast>(arg)))); } template = 0> XSIMD_INLINE batch neg(batch const& arg, requires_arch) noexcept { - return svreinterpret_u64(svneg_x(detail_sve::ptrue(), svreinterpret_s64(static_cast>(arg)))); + return svreinterpret_u64(svneg_x(detail::svptrue(), svreinterpret_s64(static_cast>(arg)))); } template = 0> XSIMD_INLINE batch neg(batch const& arg, requires_arch) noexcept { - return svneg_x(detail_sve::ptrue(), arg); + return svneg_x(detail::svptrue(), arg); } // abs @@ -381,21 +372,21 @@ namespace xsimd template = 0> XSIMD_INLINE batch abs(batch const& arg, requires_arch) noexcept { - return svabs_x(detail_sve::ptrue(), arg); + return svabs_x(detail::svptrue(), arg); } // fma: x * y + z template = 0> XSIMD_INLINE batch fma(batch const& x, batch const& y, batch const& z, requires_arch) noexcept { - return svmad_x(detail_sve::ptrue(), x, y, z); + return svmad_x(detail::svptrue(), x, y, z); } // fnma: z - x * y template = 0> XSIMD_INLINE batch fnma(batch const& x, batch const& y, batch const& z, requires_arch) noexcept { - return svmsb_x(detail_sve::ptrue(), x, y, z); + return svmsb_x(detail::svptrue(), x, y, z); } // fms: x * y - z @@ -420,7 +411,7 @@ namespace xsimd template = 0> XSIMD_INLINE batch bitwise_and(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svand_x(detail_sve::ptrue(), lhs, rhs); + return svand_x(detail::svptrue(), lhs, rhs); } template @@ -428,7 +419,7 @@ namespace xsimd { const auto lhs_bits = svreinterpret_u32(static_cast>(lhs)); const auto rhs_bits = svreinterpret_u32(static_cast>(rhs)); - const auto result_bits = svand_x(detail_sve::ptrue(), lhs_bits, rhs_bits); + const auto result_bits = svand_x(detail::svptrue(), lhs_bits, rhs_bits); return svreinterpret_f32(result_bits); } @@ -437,21 +428,21 @@ namespace xsimd { const auto lhs_bits = svreinterpret_u64(static_cast>(lhs)); const auto rhs_bits = svreinterpret_u64(static_cast>(rhs)); - const auto result_bits = svand_x(detail_sve::ptrue(), lhs_bits, rhs_bits); + const auto result_bits = svand_x(detail::svptrue(), lhs_bits, rhs_bits); return svreinterpret_f64(result_bits); } template = 0> XSIMD_INLINE batch_bool bitwise_and(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept { - return svand_z(detail_sve::ptrue(), lhs, rhs); + return svand_z(detail::svptrue(), lhs, rhs); } // bitwise_andnot template = 0> XSIMD_INLINE batch bitwise_andnot(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svbic_x(detail_sve::ptrue(), lhs, rhs); + return svbic_x(detail::svptrue(), lhs, rhs); } template @@ -459,7 +450,7 @@ namespace xsimd { const auto lhs_bits = svreinterpret_u32(static_cast>(lhs)); const auto rhs_bits = svreinterpret_u32(static_cast>(rhs)); - const auto result_bits = svbic_x(detail_sve::ptrue(), lhs_bits, rhs_bits); + const auto result_bits = svbic_x(detail::svptrue(), lhs_bits, rhs_bits); return svreinterpret_f32(result_bits); } @@ -468,21 +459,21 @@ namespace xsimd { const auto lhs_bits = svreinterpret_u64(static_cast>(lhs)); const auto rhs_bits = svreinterpret_u64(static_cast>(rhs)); - const auto result_bits = svbic_x(detail_sve::ptrue(), lhs_bits, rhs_bits); + const auto result_bits = svbic_x(detail::svptrue(), lhs_bits, rhs_bits); return svreinterpret_f64(result_bits); } template = 0> XSIMD_INLINE batch_bool bitwise_andnot(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept { - return svbic_z(detail_sve::ptrue(), lhs, rhs); + return svbic_z(detail::svptrue(), lhs, rhs); } // bitwise_or template = 0> XSIMD_INLINE batch bitwise_or(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svorr_x(detail_sve::ptrue(), lhs, rhs); + return svorr_x(detail::svptrue(), lhs, rhs); } template @@ -490,7 +481,7 @@ namespace xsimd { const auto lhs_bits = svreinterpret_u32(static_cast>(lhs)); const auto rhs_bits = svreinterpret_u32(static_cast>(rhs)); - const auto result_bits = svorr_x(detail_sve::ptrue(), lhs_bits, rhs_bits); + const auto result_bits = svorr_x(detail::svptrue(), lhs_bits, rhs_bits); return svreinterpret_f32(result_bits); } @@ -499,21 +490,21 @@ namespace xsimd { const auto lhs_bits = svreinterpret_u64(static_cast>(lhs)); const auto rhs_bits = svreinterpret_u64(static_cast>(rhs)); - const auto result_bits = svorr_x(detail_sve::ptrue(), lhs_bits, rhs_bits); + const auto result_bits = svorr_x(detail::svptrue(), lhs_bits, rhs_bits); return svreinterpret_f64(result_bits); } template = 0> XSIMD_INLINE batch_bool bitwise_or(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept { - return svorr_z(detail_sve::ptrue(), lhs, rhs); + return svorr_z(detail::svptrue(), lhs, rhs); } // bitwise_xor template = 0> XSIMD_INLINE batch bitwise_xor(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return sveor_x(detail_sve::ptrue(), lhs, rhs); + return sveor_x(detail::svptrue(), lhs, rhs); } template @@ -521,7 +512,7 @@ namespace xsimd { const auto lhs_bits = svreinterpret_u32(static_cast>(lhs)); const auto rhs_bits = svreinterpret_u32(static_cast>(rhs)); - const auto result_bits = sveor_x(detail_sve::ptrue(), lhs_bits, rhs_bits); + const auto result_bits = sveor_x(detail::svptrue(), lhs_bits, rhs_bits); return svreinterpret_f32(result_bits); } @@ -530,28 +521,28 @@ namespace xsimd { const auto lhs_bits = svreinterpret_u64(static_cast>(lhs)); const auto rhs_bits = svreinterpret_u64(static_cast>(rhs)); - const auto result_bits = sveor_x(detail_sve::ptrue(), lhs_bits, rhs_bits); + const auto result_bits = sveor_x(detail::svptrue(), lhs_bits, rhs_bits); return svreinterpret_f64(result_bits); } template = 0> XSIMD_INLINE batch_bool bitwise_xor(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept { - return sveor_z(detail_sve::ptrue(), lhs, rhs); + return sveor_z(detail::svptrue(), lhs, rhs); } // bitwise_not template = 0> XSIMD_INLINE batch bitwise_not(batch const& arg, requires_arch) noexcept { - return svnot_x(detail_sve::ptrue(), arg); + return svnot_x(detail::svptrue(), arg); } template XSIMD_INLINE batch bitwise_not(batch const& arg, requires_arch) noexcept { const auto arg_bits = svreinterpret_u32(static_cast>(arg)); - const auto result_bits = svnot_x(detail_sve::ptrue(), arg_bits); + const auto result_bits = svnot_x(detail::svptrue(), arg_bits); return svreinterpret_f32(result_bits); } @@ -559,14 +550,14 @@ namespace xsimd XSIMD_INLINE batch bitwise_not(batch const& arg, requires_arch) noexcept { const auto arg_bits = svreinterpret_u64(static_cast>(arg)); - const auto result_bits = svnot_x(detail_sve::ptrue(), arg_bits); + const auto result_bits = svnot_x(detail::svptrue(), arg_bits); return svreinterpret_f64(result_bits); } template = 0> XSIMD_INLINE batch_bool bitwise_not(batch_bool const& arg, requires_arch) noexcept { - return svnot_z(detail_sve::ptrue(), arg); + return svnot_z(detail::svptrue(), arg); } /********** @@ -615,13 +606,13 @@ namespace xsimd { constexpr std::size_t size = sizeof(typename batch::value_type) * 8; assert(0 <= n && static_cast(n) < size && "index in bounds"); - return svlsl_x(detail_sve::ptrue(), arg, n); + return svlsl_x(detail::svptrue(), arg, n); } template = 0> XSIMD_INLINE batch bitwise_lshift(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svlsl_x(detail_sve::ptrue(), lhs, detail_sve::to_unsigned_batch(rhs)); + return svlsl_x(detail::svptrue(), lhs, detail_sve::to_unsigned_batch(rhs)); } // bitwise_rshift @@ -630,13 +621,13 @@ namespace xsimd { constexpr std::size_t size = sizeof(typename batch::value_type) * 8; assert(0 <= n && static_cast(n) < size && "index in bounds"); - return svlsr_x(detail_sve::ptrue(), arg, static_cast(n)); + return svlsr_x(detail::svptrue(), arg, static_cast(n)); } template = 0> XSIMD_INLINE batch bitwise_rshift(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svlsr_x(detail_sve::ptrue(), lhs, rhs); + return svlsr_x(detail::svptrue(), lhs, rhs); } template = 0> @@ -644,13 +635,13 @@ namespace xsimd { constexpr std::size_t size = sizeof(typename batch::value_type) * 8; assert(0 <= n && static_cast(n) < size && "index in bounds"); - return svasr_x(detail_sve::ptrue(), arg, static_cast>(n)); + return svasr_x(detail::svptrue(), arg, static_cast>(n)); } template = 0> XSIMD_INLINE batch bitwise_rshift(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svasr_x(detail_sve::ptrue(), lhs, detail_sve::to_unsigned_batch(rhs)); + return svasr_x(detail::svptrue(), lhs, detail_sve::to_unsigned_batch(rhs)); } /************** @@ -662,21 +653,21 @@ namespace xsimd XSIMD_INLINE V reduce_add(batch const& arg, requires_arch) noexcept { // sve integer reduction results are promoted to 64 bits - return static_cast(svaddv(detail_sve::ptrue(), arg)); + return static_cast(svaddv(detail::svptrue(), arg)); } // reduce_max template = 0> XSIMD_INLINE T reduce_max(batch const& arg, requires_arch) noexcept { - return svmaxv(detail_sve::ptrue(), arg); + return svmaxv(detail::svptrue(), arg); } // reduce_min template = 0> XSIMD_INLINE T reduce_min(batch const& arg, requires_arch) noexcept { - return svminv(detail_sve::ptrue(), arg); + return svminv(detail::svptrue(), arg); } // haddp @@ -689,7 +680,7 @@ namespace xsimd { sums[i] = reduce_add(row[i], sve {}); } - return svld1(detail_sve::ptrue(), sums); + return svld1(detail::svptrue(), sums); } /*************** @@ -700,55 +691,55 @@ namespace xsimd template = 0> XSIMD_INLINE batch_bool eq(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svcmpeq(detail_sve::ptrue(), lhs, rhs); + return svcmpeq(detail::svptrue(), lhs, rhs); } template = 0> XSIMD_INLINE batch_bool eq(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept { - const auto neq_result = sveor_z(detail_sve::ptrue(), lhs, rhs); - return svnot_z(detail_sve::ptrue(), neq_result); + const auto neq_result = sveor_z(detail::svptrue(), lhs, rhs); + return svnot_z(detail::svptrue(), neq_result); } // neq template = 0> XSIMD_INLINE batch_bool neq(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svcmpne(detail_sve::ptrue(), lhs, rhs); + return svcmpne(detail::svptrue(), lhs, rhs); } template = 0> XSIMD_INLINE batch_bool neq(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept { - return sveor_z(detail_sve::ptrue(), lhs, rhs); + return sveor_z(detail::svptrue(), lhs, rhs); } // lt template = 0> XSIMD_INLINE batch_bool lt(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svcmplt(detail_sve::ptrue(), lhs, rhs); + return svcmplt(detail::svptrue(), lhs, rhs); } // le template = 0> XSIMD_INLINE batch_bool le(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svcmple(detail_sve::ptrue(), lhs, rhs); + return svcmple(detail::svptrue(), lhs, rhs); } // gt template = 0> XSIMD_INLINE batch_bool gt(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svcmpgt(detail_sve::ptrue(), lhs, rhs); + return svcmpgt(detail::svptrue(), lhs, rhs); } // ge template = 0> XSIMD_INLINE batch_bool ge(batch const& lhs, batch const& rhs, requires_arch) noexcept { - return svcmpge(detail_sve::ptrue(), lhs, rhs); + return svcmpge(detail::svptrue(), lhs, rhs); } /*************** @@ -890,7 +881,7 @@ namespace xsimd template = 0> XSIMD_INLINE batch sqrt(batch const& arg, requires_arch) noexcept { - return svsqrt_x(detail_sve::ptrue(), arg); + return svsqrt_x(detail::svptrue(), arg); } // reciprocal @@ -912,37 +903,37 @@ namespace xsimd template = 0> XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept { - return svcvt_f32_x(detail_sve::ptrue(), arg); + return svcvt_f32_x(detail::svptrue(), arg); } template = 0> XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept { - return svcvt_f64_x(detail_sve::ptrue(), arg); + return svcvt_f64_x(detail::svptrue(), arg); } template XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept { - return svcvt_s32_x(detail_sve::ptrue(), arg); + return svcvt_s32_x(detail::svptrue(), arg); } template XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept { - return svcvt_u32_x(detail_sve::ptrue(), arg); + return svcvt_u32_x(detail::svptrue(), arg); } template XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept { - return svcvt_s64_x(detail_sve::ptrue(), arg); + return svcvt_s64_x(detail::svptrue(), arg); } template XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept { - return svcvt_u64_x(detail_sve::ptrue(), arg); + return svcvt_u64_x(detail::svptrue(), arg); } } // namespace XSIMD_SVE_NAMESPACE } // namespace detail_sve @@ -972,7 +963,7 @@ namespace xsimd using U = as_unsigned_integer_t; const auto values = detail_sve::sve_vector_type { static_cast(args)... }; const auto zero = broadcast(static_cast(0), sve {}); - return svcmpne(detail_sve::ptrue(), values, zero); + return svcmpne(detail::svptrue(), values, zero); } // insert @@ -996,7 +987,7 @@ namespace xsimd { // create a predicate with only the I-th lane activated const auto iota = detail_sve::iota(); - const auto index_predicate = svcmpeq(detail_sve::ptrue(), iota, static_cast>(I)); + const auto index_predicate = svcmpeq(detail::svptrue(), iota, static_cast>(I)); return svsel(index_predicate, static_cast>(broadcast(val, sve {})), static_cast>(arg)); } @@ -1181,29 +1172,29 @@ namespace xsimd template = 0> XSIMD_INLINE batch nearbyint(batch const& arg, requires_arch) noexcept { - return svrintx_x(detail_sve::ptrue(), arg); + return svrintx_x(detail::svptrue(), arg); } // nearbyint_as_int template XSIMD_INLINE batch nearbyint_as_int(batch const& arg, requires_arch) noexcept { - const auto nearest = svrintx_x(detail_sve::ptrue(), arg); - return svcvt_s32_x(detail_sve::ptrue(), nearest); + const auto nearest = svrintx_x(detail::svptrue(), arg); + return svcvt_s32_x(detail::svptrue(), nearest); } template XSIMD_INLINE batch nearbyint_as_int(batch const& arg, requires_arch) noexcept { - const auto nearest = svrintx_x(detail_sve::ptrue(), arg); - return svcvt_s64_x(detail_sve::ptrue(), nearest); + const auto nearest = svrintx_x(detail::svptrue(), arg); + return svcvt_s64_x(detail::svptrue(), nearest); } // ldexp template = 0> XSIMD_INLINE batch ldexp(const batch& x, const batch, A>& exp, requires_arch) noexcept { - return svscale_x(detail_sve::ptrue(), x, exp); + return svscale_x(detail::svptrue(), x, exp); } } // namespace kernel diff --git a/include/xsimd/types/xsimd_all_registers.hpp b/include/xsimd/types/xsimd_all_registers.hpp index 75d771752..7dc66f39e 100644 --- a/include/xsimd/types/xsimd_all_registers.hpp +++ b/include/xsimd/types/xsimd_all_registers.hpp @@ -10,6 +10,7 @@ * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ +#include "./xsimd_sve_register.hpp" #include "./xsimd_avx2_register.hpp" #include "./xsimd_avx512bw_register.hpp" #include "./xsimd_avx512cd_register.hpp" @@ -31,14 +32,11 @@ #include "./xsimd_fma3_sse_register.hpp" #include "./xsimd_fma4_register.hpp" #include "./xsimd_i8mm_neon64_register.hpp" -#include "./xsimd_neon64_register.hpp" -#include "./xsimd_neon_register.hpp" #include "./xsimd_rvv_register.hpp" #include "./xsimd_sse2_register.hpp" #include "./xsimd_sse3_register.hpp" #include "./xsimd_sse4_1_register.hpp" #include "./xsimd_sse4_2_register.hpp" -#include "./xsimd_sve_register.hpp" #include "./xsimd_vsx_register.hpp" #include "./xsimd_vxe_register.hpp" #include "./xsimd_wasm_register.hpp" diff --git a/include/xsimd/types/xsimd_arm_registers.hpp b/include/xsimd/types/xsimd_arm_registers.hpp new file mode 100644 index 000000000..1af981f70 --- /dev/null +++ b/include/xsimd/types/xsimd_arm_registers.hpp @@ -0,0 +1,16 @@ +/*************************************************************************** + * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * + * Martin Renou * + * Copyright (c) QuantStack * + * Copyright (c) Serge Guelton * + * Copyright (c) Marco Barbone * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#include "./xsimd_i8mm_neon64_register.hpp" +#include "./xsimd_neon64_register.hpp" +#include "./xsimd_neon_register.hpp" +#include "./xsimd_sve_register.hpp" diff --git a/include/xsimd/types/xsimd_batch.hpp b/include/xsimd/types/xsimd_batch.hpp index e97dd24e0..5c820e05c 100644 --- a/include/xsimd/types/xsimd_batch.hpp +++ b/include/xsimd/types/xsimd_batch.hpp @@ -16,6 +16,7 @@ #include "../config/xsimd_config.hpp" #include "../config/xsimd_macros.hpp" #include "../memory/xsimd_alignment.hpp" +#include "../v15/kernel_fwd.hpp" #include "./xsimd_batch_fwd.hpp" #include "./xsimd_utils.hpp" @@ -1049,7 +1050,7 @@ namespace xsimd XSIMD_INLINE batch& batch::operator+=(batch const& other) noexcept { detail::static_check_supported_config(); - return *this = kernel::add(*this, other, A {}); + return *this = kernel::add(*this, other); } template diff --git a/include/xsimd/types/xsimd_sve_register.hpp b/include/xsimd/types/xsimd_sve_register.hpp index e4920d13b..4c01b9b49 100644 --- a/include/xsimd/types/xsimd_sve_register.hpp +++ b/include/xsimd/types/xsimd_sve_register.hpp @@ -211,6 +211,23 @@ namespace xsimd #else using sve = detail::sve<0xFFFFFFFF>; #endif + + namespace detail + { + template + struct is_sve : std::false_type + { + }; + + template + struct is_sve> : std::true_type + { + }; + } + + template + inline constexpr bool is_sve_v = detail::is_sve::value; + } // namespace xsimd #endif diff --git a/include/xsimd/v15/arithmetic/arm.hpp b/include/xsimd/v15/arithmetic/arm.hpp new file mode 100644 index 000000000..f0b6bf3d7 --- /dev/null +++ b/include/xsimd/v15/arithmetic/arm.hpp @@ -0,0 +1,91 @@ +/**************************************************************************** + * Copyright (c) xsimd contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ARITHMETIC_ARM_HPP +#define XSIMD_ARITHMETIC_ARM_HPP + +#include "../../arch/utils/sve.hpp" +#include "../../config/xsimd_macros.hpp" +#include "../../types/xsimd_arm_registers.hpp" +#include "../../types/xsimd_batch.hpp" +#include "../../utils/xsimd_type_traits.hpp" +#include "../kernel_fwd.hpp" + +#include + +namespace xsimd::kernel +{ + namespace detail + { + template + XSIMD_INLINE batch neon_vaddq(batch a, batch b) noexcept + { + using TN = map_to_sized_type_t; + if constexpr (std::is_same_v) + { + return { vaddq_u8(a.to_native(), b.to_native()) }; + } + else if constexpr (std::is_same_v) + { + return { vaddq_s8(a.to_native(), b.to_native()) }; + } + else if constexpr (std::is_same_v) + { + return { vaddq_u16(a.to_native(), b.to_native()) }; + } + else if constexpr (std::is_same_v) + { + return { vaddq_s16(a.to_native(), b.to_native()) }; + } + else if constexpr (std::is_same_v) + { + return { vaddq_u32(a.to_native(), b.to_native()) }; + } + else if constexpr (std::is_same_v) + { + return { vaddq_s32(a.to_native(), b.to_native()) }; + } + else if constexpr (std::is_same_v) + { + return { vaddq_u64(a.to_native(), b.to_native()) }; + } + else if constexpr (std::is_same_v) + { + return { vaddq_s64(a.to_native(), b.to_native()) }; + } + else if constexpr (std::is_same_v) + { + return { vaddq_f32(a.to_native(), b.to_native()) }; + } + else if constexpr (std::is_same_v && !std::is_same_v) + { + // Unavailable in neon neon arm v7 + return { vaddq_f64(a.to_native(), b.to_native()) }; + } + else + { + static_assert(false, "unsupported data type"); + } + } + } + + template + XSIMD_INLINE batch add(batch lhs, batch rhs) noexcept + { + if constexpr (is_sve_v) + { + return svadd_x(detail::svptrue(), lhs, rhs); + } + else if constexpr (std::is_same_v || std::is_same_v) + { + return detail::neon_vaddq(lhs, rhs); + } + } +} + +#endif diff --git a/include/xsimd/v15/kernel_fwd.hpp b/include/xsimd/v15/kernel_fwd.hpp new file mode 100644 index 000000000..20f38c5ef --- /dev/null +++ b/include/xsimd/v15/kernel_fwd.hpp @@ -0,0 +1,20 @@ +/**************************************************************************** + * Copyright (c) xsimd contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_KERNEL_FWD_HPP +#define XSIMD_KERNEL_FWD_HPP + +#include "../types/xsimd_batch_fwd.hpp" + +namespace xsimd::kernel +{ + template + batch add(batch lhs, batch rhs) noexcept; +} + +#endif From 281df48ff9f6a98a73328a2d8757caad954dc7ae Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Wed, 5 Aug 2026 11:38:44 +0200 Subject: [PATCH 2/2] Move add to v15 on x86 --- .../arch/common/xsimd_common_arithmetic.hpp | 5 +- include/xsimd/arch/utils/x86.hpp | 177 +++++++++++++ include/xsimd/arch/xsimd_avx.hpp | 230 +++++++---------- include/xsimd/arch/xsimd_avx2.hpp | 42 +--- include/xsimd/arch/xsimd_avx512bw.hpp | 18 -- include/xsimd/arch/xsimd_avx512f.hpp | 238 +++++------------- include/xsimd/arch/xsimd_isa.hpp | 4 +- include/xsimd/arch/xsimd_sse2.hpp | 28 +-- include/xsimd/types/xsimd_all_registers.hpp | 28 +-- include/xsimd/types/xsimd_x86_registers.hpp | 36 +++ include/xsimd/v15/arithmetic/x86.hpp | 187 ++++++++++++++ 11 files changed, 575 insertions(+), 418 deletions(-) create mode 100644 include/xsimd/arch/utils/x86.hpp create mode 100644 include/xsimd/types/xsimd_x86_registers.hpp create mode 100644 include/xsimd/v15/arithmetic/x86.hpp diff --git a/include/xsimd/arch/common/xsimd_common_arithmetic.hpp b/include/xsimd/arch/common/xsimd_common_arithmetic.hpp index 6e2b41812..5eb7cda32 100644 --- a/include/xsimd/arch/common/xsimd_common_arithmetic.hpp +++ b/include/xsimd/arch/common/xsimd_common_arithmetic.hpp @@ -13,6 +13,7 @@ #define XSIMD_COMMON_ARITHMETIC_HPP #include "../../types/xsimd_batch_constant.hpp" +#include "../../v15/kernel_fwd.hpp" #include "./xsimd_common_details.hpp" #include @@ -363,7 +364,7 @@ namespace xsimd template XSIMD_INLINE batch sadd(batch const& self, batch const& other, requires_arch) noexcept { - return add(self, other); // no saturated arithmetic on floating point numbers + return kernel::add(self, other); // no saturated arithmetic on floating point numbers } template >*/> XSIMD_INLINE batch sadd(batch const& self, batch const& other, requires_arch) noexcept @@ -384,7 +385,7 @@ namespace xsimd template XSIMD_INLINE batch sadd(batch const& self, batch const& other, requires_arch) noexcept { - return add(self, other); // no saturated arithmetic on floating point numbers + return kernel::add(self, other); // no saturated arithmetic on floating point numbers } // ssub diff --git a/include/xsimd/arch/utils/x86.hpp b/include/xsimd/arch/utils/x86.hpp new file mode 100644 index 000000000..9e0bd9c69 --- /dev/null +++ b/include/xsimd/arch/utils/x86.hpp @@ -0,0 +1,177 @@ +/**************************************************************************** + * Copyright (c) xsimd contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ARCH_UTILS_AVX_HPP +#define XSIMD_ARCH_UTILS_AVX_HPP + +#include "../../config/xsimd_macros.hpp" +#include "../../types/xsimd_batch.hpp" +#include "../../types/xsimd_x86_registers.hpp" + +#include + +namespace xsimd::kernel::detail +{ + template + using half_batch_t = make_sized_batch_t::size / 2>; + + template + using half_arch_t = typename half_batch_t::arch_type; + + template > + XSIMD_INLINE batch lower_half(batch self) noexcept + { + if constexpr (sizeof(self) == 64) + { + if constexpr (std::is_same_v) + { + return _mm512_castps512_ps256(self); + } + else if constexpr (std::is_same_v) + { + return _mm512_castpd512_pd256(self); + } + else if constexpr (std::is_integral_v) + { + return _mm512_castsi512_si256(self); + } + } + else if constexpr (sizeof(self) == 32) + { + if constexpr (sizeof(self) == 32 && std::is_same_v) + { + return _mm256_castps256_ps128(self); + } + else if constexpr (sizeof(self) == 32 && std::is_same_v) + { + return _mm256_castpd256_pd128(self); + } + else if constexpr (sizeof(self) == 32 && std::is_integral_v) + { + return _mm256_castsi256_si128(self); + } + } + else + { + static_assert(false, "unsupported architecture conversion"); + } + } + + template > + XSIMD_INLINE batch upper_half(batch self) noexcept + { + if constexpr (sizeof(self) == 64) + { + if constexpr (std::is_same_v) + { + // _mm512_extractf32x8_ps is AVX512DQ but the casts here are a noop + return _mm256_castsi256_ps(_mm512_extracti64x4_epi64(_mm512_castps_si512(self), 1)); + } + else if constexpr (std::is_same_v) + { + return _mm512_extractf64x4_pd(self, 1); + } + else if constexpr (std::is_integral_v) + { + return _mm512_extracti64x4_epi64(self, 1); + } + } + else if constexpr (sizeof(self) == 32) + { + if constexpr (std::is_same_v) + { + return _mm256_extractf128_ps(self, 1); + } + else if constexpr (std::is_same_v) + { + return _mm256_extractf128_pd(self, 1); + } + else if constexpr (std::is_integral_v) + { + return _mm256_extractf128_si256(self, 1); + } + } + else + { + static_assert(false, "unsupported architecture conversion"); + } + } + + template > + XSIMD_INLINE batch merge_halves(batch low, batch high) noexcept + { + if constexpr (sizeof(batch) == 64) + { + if constexpr (std::is_same_v) + { + // _mm512_insertf32x8 is AVX512DQ but the casts here are a noop + auto const ld = _mm256_castps_pd(low); + auto const lh = _mm256_castps_pd(high); + return _mm512_castpd_ps(_mm512_insertf64x4(_mm512_castpd256_pd512(ld), lh, 1)); + } + else if constexpr (std::is_same_v) + { + return _mm512_insertf64x4(_mm512_castpd256_pd512(low), high, 1); + } + else if constexpr (std::is_integral_v) + { + return _mm512_inserti64x4(_mm512_castsi256_si512(low), high, 1); + } + } + if constexpr (sizeof(batch) == 32) + { + if constexpr (std::is_same_v) + { + return _mm256_insertf128_ps(_mm256_castps128_ps256(low), high, 1); + } + else if constexpr (std::is_same_v) + { + return _mm256_insertf128_pd(_mm256_castpd128_pd256(low), high, 1); + } + else if constexpr (std::is_integral_v) + { + return _mm256_insertf128_si256(_mm256_castsi128_si256(low), high, 1); + } + } + else + { + static_assert(false, "unsupported architecture conversion"); + } + } + + template + XSIMD_INLINE batch apply_on_halves_with_arch(F&& f, batch self) noexcept + { + auto low = f(lower_half(self)); + auto high = f(upper_half(self)); + return merge_halves(low, high); + } + + template + XSIMD_INLINE batch apply_on_halves_with_arch(F&& f, batch lhs, batch rhs) noexcept + { + auto low = f(lower_half(lhs), lower_half(rhs)); + auto high = f(upper_half(lhs), upper_half(rhs)); + return merge_halves(low, high); + } + + template + XSIMD_INLINE batch apply_on_halves(F&& f, batch self) noexcept + { + using A1 = half_arch_t; + return apply_on_halves_with_arch(std::forward(f), self); + } + + template + XSIMD_INLINE batch apply_on_halves(F&& f, batch lhs, batch rhs) noexcept + { + using A1 = half_arch_t; + return apply_on_halves_with_arch(std::forward(f), lhs, rhs); + } +} +#endif diff --git a/include/xsimd/arch/xsimd_avx.hpp b/include/xsimd/arch/xsimd_avx.hpp index 814452cea..ce2e086fc 100644 --- a/include/xsimd/arch/xsimd_avx.hpp +++ b/include/xsimd/arch/xsimd_avx.hpp @@ -13,6 +13,7 @@ #ifndef XSIMD_AVX_HPP #define XSIMD_AVX_HPP +#include "../arch/utils/x86.hpp" #include "../types/xsimd_avx_register.hpp" #include "../types/xsimd_batch_constant.hpp" @@ -38,66 +39,31 @@ namespace xsimd namespace detail { - XSIMD_INLINE __m128i lower_half(__m256i self) noexcept - { - return _mm256_castsi256_si128(self); - } - XSIMD_INLINE __m128 lower_half(__m256 self) noexcept - { - return _mm256_castps256_ps128(self); - } - XSIMD_INLINE __m128d lower_half(__m256d self) noexcept - { - return _mm256_castpd256_pd128(self); - } - XSIMD_INLINE __m128i upper_half(__m256i self) noexcept - { - return _mm256_extractf128_si256(self, 1); - } - XSIMD_INLINE __m128 upper_half(__m256 self) noexcept - { - return _mm256_extractf128_ps(self, 1); - } - XSIMD_INLINE __m128d upper_half(__m256d self) noexcept - { - return _mm256_extractf128_pd(self, 1); - } - XSIMD_INLINE __m256i merge_sse(__m128i low, __m128i high) noexcept - { - return _mm256_insertf128_si256(_mm256_castsi128_si256(low), high, 1); - } - XSIMD_INLINE __m256 merge_sse(__m128 low, __m128 high) noexcept - { - return _mm256_insertf128_ps(_mm256_castps128_ps256(low), high, 1); - } - XSIMD_INLINE __m256d merge_sse(__m128d low, __m128d high) noexcept + // Half arch cannot be deduced for bools, since on AVX512 archs a batch_bool is a mask. + using half_bool_arch = sse4_2; + + template + XSIMD_INLINE batch_bool lower_bool_half(batch_bool self) noexcept { - return _mm256_insertf128_pd(_mm256_castpd128_pd256(low), high, 1); + return lower_half(batch(self.data)).data; } - template - XSIMD_INLINE __m256i fwd_to_sse(F f, __m256i self) noexcept + + template + XSIMD_INLINE batch_bool upper_bool_half(batch_bool self) noexcept { - __m128i self_low = lower_half(self), self_high = upper_half(self); - __m128i res_low = f(self_low); - __m128i res_high = f(self_high); - return merge_sse(res_low, res_high); + return upper_half(batch(self.data)).data; } - template - XSIMD_INLINE __m256i fwd_to_sse(F f, __m256i self, __m256i other) noexcept + + template + XSIMD_INLINE batch_bool apply_on_bool_halves(F&& f, batch_bool self) noexcept { - __m128i self_low = lower_half(self), self_high = upper_half(self), - other_low = lower_half(other), other_high = upper_half(other); - __m128i res_low = f(self_low, other_low); - __m128i res_high = f(self_high, other_high); - return merge_sse(res_low, res_high); + return apply_on_halves_with_arch(std::forward(f), batch(self.data)).data; } - template - XSIMD_INLINE __m256i fwd_to_sse(F f, __m256i self, int32_t other) noexcept + + template + XSIMD_INLINE batch_bool apply_on_bool_halves(F&& f, batch_bool lhs, batch_bool rhs) noexcept { - __m128i self_low = lower_half(self), self_high = upper_half(self); - __m128i res_low = f(self_low, other); - __m128i res_high = f(self_high, other); - return merge_sse(res_low, res_high); + return apply_on_halves_with_arch(std::forward(f), batch(lhs.data), batch(rhs.data)).data; } } @@ -115,25 +81,6 @@ namespace xsimd return _mm256_andnot_pd(sign_mask, self); } - // add - template >> - XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept - { - return detail::fwd_to_sse([](__m128i s, __m128i o) noexcept - { return add(batch(s), batch(o)); }, - self, other); - } - template - XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept - { - return _mm256_add_ps(self, other); - } - template - XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept - { - return _mm256_add_pd(self, other); - } - // all template XSIMD_INLINE bool all(batch_bool const& self, requires_arch) noexcept @@ -201,16 +148,16 @@ namespace xsimd template >> XSIMD_INLINE batch bitwise_and(batch const& self, batch const& other, requires_arch) noexcept { - return detail::fwd_to_sse([](__m128i s, __m128i o) noexcept - { return bitwise_and(batch(s), batch(o)); }, - self, other); + return detail::apply_on_halves([](auto s, auto o) noexcept + { return bitwise_and(s, o); }, + self, other); } template >> XSIMD_INLINE batch_bool bitwise_and(batch_bool const& self, batch_bool const& other, requires_arch) noexcept { - return detail::fwd_to_sse([](__m128i s, __m128i o) noexcept - { return bitwise_and(batch(s), batch(o)); }, - self, other); + return detail::apply_on_bool_halves([](auto s, auto o) noexcept + { return bitwise_and(s, o); }, + self, other); } // bitwise_andnot @@ -239,41 +186,41 @@ namespace xsimd template >> XSIMD_INLINE batch bitwise_andnot(batch const& self, batch const& other, requires_arch) noexcept { - return detail::fwd_to_sse([](__m128i s, __m128i o) noexcept - { return bitwise_andnot(batch(s), batch(o)); }, - self, other); + return detail::apply_on_halves([](auto s, auto o) noexcept + { return bitwise_andnot(s, o); }, + self, other); } template >> XSIMD_INLINE batch_bool bitwise_andnot(batch_bool const& self, batch_bool const& other, requires_arch) noexcept { - return detail::fwd_to_sse([](__m128i s, __m128i o) noexcept - { return bitwise_andnot(batch(s), batch(o)); }, - self, other); + return detail::apply_on_bool_halves([](auto s, auto o) noexcept + { return bitwise_andnot(s, o); }, + self, other); } // bitwise_lshift template >> XSIMD_INLINE batch bitwise_lshift(batch const& self, int32_t other, requires_arch) noexcept { - return detail::fwd_to_sse([](__m128i s, int32_t o) noexcept - { return bitwise_lshift(batch(s), o, sse4_2 {}); }, - self, other); + return detail::apply_on_halves([other](auto s) noexcept + { return bitwise_lshift(s, other); }, + self); } // bitwise_not template >> XSIMD_INLINE batch bitwise_not(batch const& self, requires_arch) noexcept { - return detail::fwd_to_sse([](__m128i s) noexcept - { return bitwise_not(batch(s), sse4_2 {}); }, - self); + return detail::apply_on_halves([](auto s) noexcept + { return bitwise_not(s); }, + self); } template >> XSIMD_INLINE batch_bool bitwise_not(batch_bool const& self, requires_arch) noexcept { - return detail::fwd_to_sse([](__m128i s) noexcept - { return bitwise_not(batch_bool(s), sse4_2 {}); }, - self); + return detail::apply_on_bool_halves([](auto s) noexcept + { return bitwise_not(s); }, + self); } // bitwise_or @@ -300,25 +247,25 @@ namespace xsimd template >> XSIMD_INLINE batch bitwise_or(batch const& self, batch const& other, requires_arch) noexcept { - return detail::fwd_to_sse([](__m128i s, __m128i o) noexcept - { return bitwise_or(batch(s), batch(o)); }, - self, other); + return detail::apply_on_halves([](auto s, auto o) noexcept + { return bitwise_or(s, o); }, + self, other); } template >> XSIMD_INLINE batch_bool bitwise_or(batch_bool const& self, batch_bool const& other, requires_arch) noexcept { - return detail::fwd_to_sse([](__m128i s, __m128i o) noexcept - { return bitwise_or(batch_bool(s), batch_bool(o)); }, - self, other); + return detail::apply_on_bool_halves([](auto s, auto o) noexcept + { return bitwise_or(s, o); }, + self, other); } // bitwise_rshift template >> XSIMD_INLINE batch bitwise_rshift(batch const& self, int32_t other, requires_arch) noexcept { - return detail::fwd_to_sse([](__m128i s, int32_t o) noexcept - { return bitwise_rshift(batch(s), o, sse4_2 {}); }, - self, other); + return detail::apply_on_halves([other](auto s) noexcept + { return bitwise_rshift(s, other); }, + self); } // bitwise_xor @@ -345,16 +292,16 @@ namespace xsimd template >> XSIMD_INLINE batch bitwise_xor(batch const& self, batch const& other, requires_arch) noexcept { - return detail::fwd_to_sse([](__m128i s, __m128i o) noexcept - { return bitwise_xor(batch(s), batch(o), sse4_2 {}); }, - self, other); + return detail::apply_on_halves([](auto s, auto o) noexcept + { return bitwise_xor(s, o); }, + self, other); } template >> - XSIMD_INLINE batch bitwise_xor(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + XSIMD_INLINE batch_bool bitwise_xor(batch_bool const& self, batch_bool const& other, requires_arch) noexcept { - return detail::fwd_to_sse([](__m128i s, __m128i o) noexcept - { return bitwise_xor(batch_bool(s), batch_bool(o), sse4_2 {}); }, - self, other); + return detail::apply_on_bool_halves([](auto s, auto o) noexcept + { return bitwise_xor(s, o); }, + self, other); } // bitwise_cast @@ -578,9 +525,10 @@ namespace xsimd template >> XSIMD_INLINE batch_bool eq(batch const& self, batch const& other, requires_arch) noexcept { - return detail::fwd_to_sse([](__m128i s, __m128i o) noexcept - { return eq(batch(s), batch(o), sse4_2 {}); }, - self, other); + return detail::apply_on_halves_with_arch([](auto s, auto o) noexcept + { return decltype(s)(eq(s, o).data); }, + self, other) + .data; } template >> @@ -1044,8 +992,8 @@ namespace xsimd template >> XSIMD_INLINE batch load_masked(T const* mem, batch_bool_constant mask, convert, Mode, requires_arch) noexcept { - constexpr size_t half_size = batch::size / 2; - using half_batch = make_sized_batch_t; + using half_batch = detail::half_batch_t; + constexpr auto half_size = half_batch::size; using half_arch = typename half_batch::arch_type; // exactly the lower 128-bit half: one plain load, upper lanes zero @@ -1072,7 +1020,7 @@ namespace xsimd const half_batch lo = half_batch::load(mem, Mode {}); constexpr auto mhi = ::xsimd::detail::upper_half(mask); const half_batch hi = load_masked(mem + half_size, mhi, convert {}, Mode {}, half_arch {}); - return detail::merge_sse(lo.data, hi.data); + return detail::merge_halves(lo, hi); } // exactly the upper 128-bit half: one plain load into the upper lanes else if constexpr (mask.suffix() == half_size) @@ -1121,7 +1069,7 @@ namespace xsimd XSIMD_INLINE void store_masked(T* mem, batch const& src, batch_bool_constant mask, Mode, requires_arch) noexcept { constexpr size_t half_size = batch::size / 2; - using half_batch = ::xsimd::make_sized_batch_t; + using half_batch = make_sized_batch_t; using half_arch = typename half_batch::arch_type; // exactly the lower 128-bit half: one plain store @@ -1255,9 +1203,10 @@ namespace xsimd template >> XSIMD_INLINE batch_bool lt(batch const& self, batch const& other, requires_arch) noexcept { - return detail::fwd_to_sse([](__m128i s, __m128i o) noexcept - { return lt(batch(s), batch(o)); }, - self, other); + return detail::apply_on_halves_with_arch([](auto s, auto o) noexcept + { return decltype(s)(lt(s, o).data); }, + self, other) + .data; } // mask @@ -1266,8 +1215,8 @@ namespace xsimd { if constexpr (sizeof(T) == 1 || sizeof(T) == 2) { - __m128i self_low = detail::lower_half(self), self_high = detail::upper_half(self); - return mask(batch_bool(self_low), sse4_2 {}) | (mask(batch_bool(self_high), sse4_2 {}) << (128 / (8 * sizeof(T)))); + using half_arch = detail::half_bool_arch; + return mask(detail::lower_bool_half(self), half_arch {}) | (mask(detail::upper_bool_half(self), half_arch {}) << (128 / (8 * sizeof(T)))); } else if constexpr (sizeof(T) == 4) { @@ -1504,15 +1453,17 @@ namespace xsimd template >> XSIMD_INLINE batch select(batch_bool const& cond, batch const& true_br, batch const& false_br, requires_arch) noexcept { - __m128i cond_low = detail::lower_half(cond), cond_hi = detail::upper_half(cond); - - __m128i true_low = detail::lower_half(true_br), true_hi = detail::upper_half(true_br); - - __m128i false_low = detail::lower_half(false_br), false_hi = detail::upper_half(false_br); + using half_arch = detail::half_bool_arch; - __m128i res_low = select(batch_bool(cond_low), batch(true_low), batch(false_low), sse4_2 {}); - __m128i res_hi = select(batch_bool(cond_hi), batch(true_hi), batch(false_hi), sse4_2 {}); - return detail::merge_sse(res_low, res_hi); + const auto res_low = select(detail::lower_bool_half(cond), + detail::lower_half(true_br), + detail::lower_half(false_br), + half_arch {}); + const auto res_hi = select(detail::upper_bool_half(cond), + detail::upper_half(true_br), + detail::upper_half(false_br), + half_arch {}); + return detail::merge_halves(res_low, res_hi); } template >> XSIMD_INLINE batch select(batch_bool_constant const&, batch const& true_br, batch const& false_br, requires_arch) noexcept @@ -1794,9 +1745,9 @@ namespace xsimd template >> XSIMD_INLINE batch sub(batch const& self, batch const& other, requires_arch) noexcept { - return detail::fwd_to_sse([](__m128i s, __m128i o) noexcept - { return sub(batch(s), batch(o)); }, - self, other); + return detail::apply_on_halves([](auto s, auto o) noexcept + { return sub(s, o); }, + self, other); } template XSIMD_INLINE batch sub(batch const& self, batch const& other, requires_arch) noexcept @@ -2088,9 +2039,9 @@ namespace xsimd transpose(tmp_hi1 + 0, tmp_hi1 + 8, sse4_2 {}); for (int i = 0; i < 8; ++i) - matrix_begin[i] = detail::merge_sse(tmp_lo0[i], tmp_hi0[i]); + matrix_begin[i] = detail::merge_halves(tmp_lo0[i], tmp_hi0[i]); for (int i = 0; i < 8; ++i) - matrix_begin[i + 8] = detail::merge_sse(tmp_lo1[i], tmp_hi1[i]); + matrix_begin[i + 8] = detail::merge_halves(tmp_lo1[i], tmp_hi1[i]); } template XSIMD_INLINE void transpose(batch* matrix_begin, batch* matrix_end, requires_arch) noexcept @@ -2124,9 +2075,9 @@ namespace xsimd transpose(tmp_hi1 + 0, tmp_hi1 + 16, sse4_2 {}); for (int i = 0; i < 16; ++i) - matrix_begin[i] = detail::merge_sse(tmp_lo0[i], tmp_hi0[i]); + matrix_begin[i] = detail::merge_halves(tmp_lo0[i], tmp_hi0[i]); for (int i = 0; i < 16; ++i) - matrix_begin[i + 16] = detail::merge_sse(tmp_lo1[i], tmp_hi1[i]); + matrix_begin[i + 16] = detail::merge_halves(tmp_lo1[i], tmp_hi1[i]); } template XSIMD_INLINE void transpose(batch* matrix_begin, batch* matrix_end, requires_arch) noexcept @@ -2277,9 +2228,10 @@ namespace xsimd template XSIMD_INLINE std::array, A>, 2> widen(batch const& x, requires_arch) noexcept { - auto pair_lo = widen(batch(detail::lower_half(x)), sse4_2 {}); - auto pair_hi = widen(batch(detail::upper_half(x)), sse4_2 {}); - return { detail::merge_sse(pair_lo[0], pair_lo[1]), detail::merge_sse(pair_hi[0], pair_hi[1]) }; + auto pair_lo = widen(detail::lower_half(x), sse4_2 {}); + auto pair_hi = widen(detail::upper_half(x), sse4_2 {}); + return { detail::merge_halves, A, sse4_2>(pair_lo[0], pair_lo[1]), + detail::merge_halves, A, sse4_2>(pair_hi[0], pair_hi[1]) }; } template XSIMD_INLINE std::array, 2> widen(batch const& x, requires_arch) noexcept diff --git a/include/xsimd/arch/xsimd_avx2.hpp b/include/xsimd/arch/xsimd_avx2.hpp index ba6825cb8..ed9b6fc69 100644 --- a/include/xsimd/arch/xsimd_avx2.hpp +++ b/include/xsimd/arch/xsimd_avx2.hpp @@ -53,32 +53,6 @@ namespace xsimd return self; } - // add - template >> - XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept - { - if constexpr (sizeof(T) == 1) - { - return _mm256_add_epi8(self, other); - } - else if constexpr (sizeof(T) == 2) - { - return _mm256_add_epi16(self, other); - } - else if constexpr (sizeof(T) == 4) - { - return _mm256_add_epi32(self, other); - } - else if constexpr (sizeof(T) == 8) - { - return _mm256_add_epi64(self, other); - } - else - { - return add(self, other, avx {}); - } - } - // avgr template >> XSIMD_INLINE batch avgr(batch const& self, batch const& other, requires_arch) noexcept @@ -348,9 +322,9 @@ namespace xsimd __m256i cmp_is_negative = _mm256_cmpgt_epi8(_mm256_setzero_si256(), self); __m256i res = _mm256_srai_epi16(self, other); return _mm256_or_si256( - detail::fwd_to_sse([](__m128i s, __m128i o) noexcept - { return bitwise_and(batch(s), batch(o), sse4_2 {}); }, - sign_mask, cmp_is_negative), + detail::apply_on_halves([](auto s, auto o) noexcept + { return bitwise_and(s, o); }, + batch(sign_mask), batch(cmp_is_negative)), _mm256_andnot_si256(sign_mask, res)); } else if constexpr (sizeof(T) == 2) @@ -400,9 +374,9 @@ namespace xsimd __m256i cmp_is_negative = _mm256_cmpgt_epi8(_mm256_setzero_si256(), self); __m256i res = _mm256_srai_epi16(self, shift); return _mm256_or_si256( - detail::fwd_to_sse([](__m128i s, __m128i o) noexcept - { return bitwise_and(batch(s), batch(o), sse4_2 {}); }, - sign_mask, cmp_is_negative), + detail::apply_on_halves([](auto s, auto o) noexcept + { return bitwise_and(s, o); }, + batch(sign_mask), batch(cmp_is_negative)), _mm256_andnot_si256(sign_mask, res)); } else if constexpr (sizeof(T) == 2) @@ -611,7 +585,7 @@ namespace xsimd { const batch low(_mm256_i32gather_pd(src, _mm256_castsi256_si128(index.data), sizeof(double))); const batch high(_mm256_i32gather_pd(src, _mm256_extractf128_si256(index.data, 1), sizeof(double))); - return detail::merge_sse(_mm256_cvtpd_ps(low.data), _mm256_cvtpd_ps(high.data)); + return detail::merge_halves(_mm256_cvtpd_ps(low.data), _mm256_cvtpd_ps(high.data)); } template = 0> @@ -621,7 +595,7 @@ namespace xsimd { const batch low(_mm256_i32gather_pd(src, _mm256_castsi256_si128(index.data), sizeof(double))); const batch high(_mm256_i32gather_pd(src, _mm256_extractf128_si256(index.data, 1), sizeof(double))); - return detail::merge_sse(_mm256_cvtpd_epi32(low.data), _mm256_cvtpd_epi32(high.data)); + return detail::merge_halves(_mm256_cvtpd_epi32(low.data), _mm256_cvtpd_epi32(high.data)); } // lt diff --git a/include/xsimd/arch/xsimd_avx512bw.hpp b/include/xsimd/arch/xsimd_avx512bw.hpp index 2d32002b9..665823efc 100644 --- a/include/xsimd/arch/xsimd_avx512bw.hpp +++ b/include/xsimd/arch/xsimd_avx512bw.hpp @@ -94,24 +94,6 @@ namespace xsimd } } - // add - template >> - XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept - { - if constexpr (sizeof(T) == 1) - { - return _mm512_add_epi8(self, other); - } - else if constexpr (sizeof(T) == 2) - { - return _mm512_add_epi16(self, other); - } - else - { - return add(self, other, avx512dq {}); - } - } - // avgr template >> XSIMD_INLINE batch avgr(batch const& self, batch const& other, requires_arch) noexcept diff --git a/include/xsimd/arch/xsimd_avx512f.hpp b/include/xsimd/arch/xsimd_avx512f.hpp index 658b7d448..2faff1146 100644 --- a/include/xsimd/arch/xsimd_avx512f.hpp +++ b/include/xsimd/arch/xsimd_avx512f.hpp @@ -12,6 +12,7 @@ #ifndef XSIMD_AVX512F_HPP #define XSIMD_AVX512F_HPP +#include "../arch/utils/x86.hpp" #include "../types/xsimd_avx512f_register.hpp" #include "../types/xsimd_batch_constant.hpp" @@ -40,71 +41,6 @@ namespace xsimd template XSIMD_INLINE void transpose(batch* matrix_begin, batch* matrix_end, requires_arch) noexcept; - namespace detail - { - XSIMD_INLINE __m256 lower_half(__m512 self) noexcept - { - return _mm512_castps512_ps256(self); - } - XSIMD_INLINE __m256d lower_half(__m512d self) noexcept - { - return _mm512_castpd512_pd256(self); - } - XSIMD_INLINE __m256i lower_half(__m512i self) noexcept - { - return _mm512_castsi512_si256(self); - } - XSIMD_INLINE __m256 upper_half(__m512 self) noexcept - { - return _mm256_castsi256_ps(_mm512_extracti64x4_epi64(_mm512_castps_si512(self), 1)); - } - XSIMD_INLINE __m256d upper_half(__m512d self) noexcept - { - return _mm512_extractf64x4_pd(self, 1); - } - XSIMD_INLINE __m256i upper_half(__m512i self) noexcept - { - return _mm512_extracti64x4_epi64(self, 1); - } - XSIMD_INLINE __m512i merge_avx(__m256i low, __m256i high) noexcept - { - return _mm512_inserti64x4(_mm512_castsi256_si512(low), high, 1); - } - XSIMD_INLINE __m512 merge_avx(__m256 low, __m256 high) noexcept - { - return _mm512_castpd_ps(_mm512_insertf64x4(_mm512_castpd256_pd512(_mm256_castps_pd(low)), _mm256_castps_pd(high), 1)); - } - XSIMD_INLINE __m512d merge_avx(__m256d low, __m256d high) noexcept - { - return _mm512_insertf64x4(_mm512_castpd256_pd512(low), high, 1); - } - - template - __m512i fwd_to_avx(F f, __m512i self) - { - __m256i self_low = lower_half(self), self_high = upper_half(self); - __m256i res_low = f(self_low); - __m256i res_high = f(self_high); - return merge_avx(res_low, res_high); - } - template - __m512i fwd_to_avx(F f, __m512i self, __m512i other) - { - __m256i self_low = lower_half(self), self_high = upper_half(self), - other_low = lower_half(other), other_high = upper_half(other); - __m256i res_low = f(self_low, other_low); - __m256i res_high = f(self_high, other_high); - return merge_avx(res_low, res_high); - } - template - __m512i fwd_to_avx(F f, __m512i self, int32_t other) - { - __m256i self_low = lower_half(self), self_high = upper_half(self); - __m256i res_low = f(self_low, other); - __m256i res_high = f(self_high, other); - return merge_avx(res_low, res_high); - } - } namespace detail { @@ -400,15 +336,15 @@ namespace xsimd if constexpr (sizeof(T) == 1) { - return detail::fwd_to_avx([](__m256i s) noexcept - { return abs(batch(s)); }, - self); + return detail::apply_on_halves([](auto s) noexcept + { return abs(s); }, + self); } else if constexpr (sizeof(T) == 2) { - return detail::fwd_to_avx([](__m256i s) noexcept - { return abs(batch(s)); }, - self); + return detail::apply_on_halves([](auto s) noexcept + { return abs(s); }, + self); } else if constexpr (sizeof(T) == 4) { @@ -425,47 +361,6 @@ namespace xsimd } } - // add - template >> - XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept - { - if constexpr (sizeof(T) == 1) - { - return detail::fwd_to_avx([](__m256i s, __m256i o) noexcept - { return add(batch(s), batch(o)); }, - self, other); - } - else if constexpr (sizeof(T) == 2) - { - return detail::fwd_to_avx([](__m256i s, __m256i o) noexcept - { return add(batch(s), batch(o)); }, - self, other); - } - else if constexpr (sizeof(T) == 4) - { - return _mm512_add_epi32(self, other); - } - else if constexpr (sizeof(T) == 8) - { - return _mm512_add_epi64(self, other); - } - else - { - assert(false && "unsupported arch/op combination"); - return {}; - } - } - template - XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept - { - return _mm512_add_ps(self, other); - } - template - XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept - { - return _mm512_add_pd(self, other); - } - // all template XSIMD_INLINE bool all(batch_bool const& self, requires_arch) noexcept @@ -558,9 +453,9 @@ namespace xsimd } else if constexpr (sizeof(T) == 2) { - return detail::fwd_to_avx([](__m256i s, int32_t o) noexcept - { return bitwise_lshift(batch(s), o, avx2 {}); }, - self, other); + return detail::apply_on_halves([other](auto s) noexcept + { return bitwise_lshift(s, other); }, + self); #if defined(XSIMD_AVX512_SHIFT_INTRINSICS_IMM_ONLY) } else if constexpr (sizeof(T) == 4) @@ -663,9 +558,9 @@ namespace xsimd } else { - return detail::fwd_to_avx([](__m256i s, int32_t o) noexcept - { return bitwise_rshift(batch(s), o, avx2 {}); }, - self, other); + return detail::apply_on_halves([other](auto s) noexcept + { return bitwise_rshift(s, other); }, + self); } } else @@ -700,9 +595,9 @@ namespace xsimd } else { - return detail::fwd_to_avx([](__m256i s, int32_t o) noexcept - { return bitwise_rshift(batch(s), o, avx2 {}); }, - self, other); + return detail::apply_on_halves([other](auto s) noexcept + { return bitwise_rshift(s, other); }, + self); } } } @@ -719,9 +614,9 @@ namespace xsimd { return _mm512_rolv_epi64(self, other); } - return detail::fwd_to_avx([](__m256i s, __m256i o) noexcept - { return rotl(batch(s), batch(o), avx2 {}); }, - self, other); + return detail::apply_on_halves([](auto s, auto o) noexcept + { return rotl(s, o); }, + self, other); } template >> XSIMD_INLINE batch rotl(batch const& self, int32_t other, requires_arch) noexcept @@ -742,9 +637,9 @@ namespace xsimd return _mm512_rol_epi64(self, count); } - return detail::fwd_to_avx([](__m256i s) noexcept - { return rotl(batch(s), avx2 {}); }, - self); + return detail::apply_on_halves([](auto s) noexcept + { return rotl(s); }, + self); } // rotr @@ -753,9 +648,9 @@ namespace xsimd { if constexpr (sizeof(T) < 4) { - return detail::fwd_to_avx([](__m256i s, __m256i o) noexcept - { return rotr(batch(s), batch(o), avx2 {}); }, - self, other); + return detail::apply_on_halves([](auto s, auto o) noexcept + { return rotr(s, o); }, + self, other); } if constexpr (std::is_unsigned_v) { @@ -783,9 +678,9 @@ namespace xsimd static_assert(count < bits, "Count must be less than the number of bits in T"); if constexpr (sizeof(T) < 4) { - return detail::fwd_to_avx([](__m256i s) noexcept - { return rotr(batch(s), avx2 {}); }, - self); + return detail::apply_on_halves([](auto s) noexcept + { return rotr(s); }, + self); } if constexpr (std::is_unsigned_v) { @@ -1205,7 +1100,7 @@ namespace xsimd { const batch low(_mm512_i32gather_pd(_mm512_castsi512_si256(index.data), src, sizeof(double))); const batch high(_mm512_i32gather_pd(_mm256_castpd_si256(_mm512_extractf64x4_pd(_mm512_castsi512_pd(index.data), 1)), src, sizeof(double))); - return detail::merge_avx(_mm512_cvtpd_ps(low.data), _mm512_cvtpd_ps(high.data)); + return detail::merge_halves(_mm512_cvtpd_ps(low.data), _mm512_cvtpd_ps(high.data)); } template = 0> @@ -1215,7 +1110,7 @@ namespace xsimd { const batch low(_mm512_i32gather_pd(_mm512_castsi512_si256(index.data), src, sizeof(double))); const batch high(_mm512_i32gather_pd(_mm256_castpd_si256(_mm512_extractf64x4_pd(_mm512_castsi512_pd(index.data), 1)), src, sizeof(double))); - return detail::merge_avx(_mm512_cvtpd_epi32(low.data), _mm512_cvtpd_epi32(high.data)); + return detail::merge_halves(_mm512_cvtpd_epi32(low.data), _mm512_cvtpd_epi32(high.data)); } // ge @@ -1718,9 +1613,9 @@ namespace xsimd } else { - return detail::fwd_to_avx([](__m256i s, __m256i o) noexcept - { return max(batch(s), batch(o)); }, - self, other); + return detail::apply_on_halves([](auto s, auto o) noexcept + { return max(s, o); }, + self, other); } } else @@ -1735,9 +1630,9 @@ namespace xsimd } else { - return detail::fwd_to_avx([](__m256i s, __m256i o) noexcept - { return max(batch(s), batch(o)); }, - self, other); + return detail::apply_on_halves([](auto s, auto o) noexcept + { return max(s, o); }, + self, other); } } } @@ -1768,9 +1663,9 @@ namespace xsimd } else { - return detail::fwd_to_avx([](__m256i s, __m256i o) noexcept - { return min(batch(s), batch(o)); }, - self, other); + return detail::apply_on_halves([](auto s, auto o) noexcept + { return min(s, o); }, + self, other); } } else @@ -1785,9 +1680,9 @@ namespace xsimd } else { - return detail::fwd_to_avx([](__m256i s, __m256i o) noexcept - { return min(batch(s), batch(o)); }, - self, other); + return detail::apply_on_halves([](auto s, auto o) noexcept + { return min(s, o); }, + self, other); } } } @@ -1812,9 +1707,9 @@ namespace xsimd } else { - return detail::fwd_to_avx([](__m256i s, __m256i o) noexcept - { return mul(batch(s), batch(o)); }, - self, other); + return detail::apply_on_halves([](auto s, auto o) noexcept + { return mul(s, o); }, + self, other); } } @@ -2092,24 +1987,24 @@ namespace xsimd __m256i cond_low = batch::load_aligned(&buffer[0]); __m256i cond_hi = batch::load_aligned(&buffer[32]); - __m256i true_low = detail::lower_half(true_br), true_hi = detail::upper_half(true_br); - __m256i false_low = detail::lower_half(false_br), false_hi = detail::upper_half(false_br); + const auto true_low = detail::lower_half(true_br), true_hi = detail::upper_half(true_br); + const auto false_low = detail::lower_half(false_br), false_hi = detail::upper_half(false_br); - __m256i res_low = select(batch_bool(cond_low), batch(true_low), batch(false_low), avx2 {}); - __m256i res_hi = select(batch_bool(cond_hi), batch(true_hi), batch(false_hi), avx2 {}); - return detail::merge_avx(res_low, res_hi); + const auto res_low = select(batch_bool(cond_low), true_low, false_low, avx2 {}); + const auto res_hi = select(batch_bool(cond_hi), true_hi, false_hi, avx2 {}); + return detail::merge_halves(res_low, res_hi); } else if constexpr (sizeof(T) == 2) { __m256i cond_low = _mm512_maskz_cvtepi32_epi16((uint64_t)cond.data & 0xFFFF, _mm512_set1_epi32(~0)); __m256i cond_hi = _mm512_maskz_cvtepi32_epi16((uint64_t)cond.data >> 16, _mm512_set1_epi32(~0)); - __m256i true_low = detail::lower_half(true_br), true_hi = detail::upper_half(true_br); - __m256i false_low = detail::lower_half(false_br), false_hi = detail::upper_half(false_br); + const auto true_low = detail::lower_half(true_br), true_hi = detail::upper_half(true_br); + const auto false_low = detail::lower_half(false_br), false_hi = detail::upper_half(false_br); - __m256i res_low = select(batch_bool(cond_low), batch(true_low), batch(false_low), avx2 {}); - __m256i res_hi = select(batch_bool(cond_hi), batch(true_hi), batch(false_hi), avx2 {}); - return detail::merge_avx(res_low, res_hi); + const auto res_low = select(batch_bool(cond_low), true_low, false_low, avx2 {}); + const auto res_hi = select(batch_bool(cond_hi), true_hi, false_hi, avx2 {}); + return detail::merge_halves(res_low, res_hi); } else if constexpr (sizeof(T) == 4) { @@ -2507,15 +2402,15 @@ namespace xsimd { if constexpr (sizeof(T) == 1) { - return detail::fwd_to_avx([](__m256i s, __m256i o) noexcept - { return sub(batch(s), batch(o)); }, - self, other); + return detail::apply_on_halves([](auto s, auto o) noexcept + { return sub(s, o); }, + self, other); } else if constexpr (sizeof(T) == 2) { - return detail::fwd_to_avx([](__m256i s, __m256i o) noexcept - { return sub(batch(s), batch(o)); }, - self, other); + return detail::apply_on_halves([](auto s, auto o) noexcept + { return sub(s, o); }, + self, other); } else if constexpr (sizeof(T) == 4) { @@ -2762,9 +2657,9 @@ namespace xsimd transpose(tmp_hi1 + 0, tmp_hi1 + 16, avx2 {}); for (int i = 0; i < 16; ++i) - matrix_begin[i] = detail::merge_avx(tmp_lo0[i], tmp_hi0[i]); + matrix_begin[i] = detail::merge_halves(tmp_lo0[i], tmp_hi0[i]); for (int i = 0; i < 16; ++i) - matrix_begin[i + 16] = detail::merge_avx(tmp_lo1[i], tmp_hi1[i]); + matrix_begin[i + 16] = detail::merge_halves(tmp_lo1[i], tmp_hi1[i]); } template XSIMD_INLINE void transpose(batch* matrix_begin, batch* matrix_end, requires_arch) noexcept @@ -2798,9 +2693,9 @@ namespace xsimd transpose(tmp_hi1 + 0, tmp_hi1 + 32, avx2 {}); for (int i = 0; i < 32; ++i) - matrix_begin[i] = detail::merge_avx(tmp_lo0[i], tmp_hi0[i]); + matrix_begin[i] = detail::merge_halves(tmp_lo0[i], tmp_hi0[i]); for (int i = 0; i < 32; ++i) - matrix_begin[i + 32] = detail::merge_avx(tmp_lo1[i], tmp_hi1[i]); + matrix_begin[i + 32] = detail::merge_halves(tmp_lo1[i], tmp_hi1[i]); } template XSIMD_INLINE void transpose(batch* matrix_begin, batch* matrix_end, requires_arch) noexcept @@ -3033,7 +2928,8 @@ namespace xsimd { auto pair_lo = widen(batch(x_lo), avx2 {}); auto pair_hi = widen(batch(x_hi), avx2 {}); - return { detail::merge_avx(pair_lo[0], pair_lo[1]), detail::merge_avx(pair_hi[0], pair_hi[1]) }; + return { detail::merge_halves, A, avx2>(pair_lo[0], pair_lo[1]), + detail::merge_halves, A, avx2>(pair_hi[0], pair_hi[1]) }; } return { lo, hi }; } diff --git a/include/xsimd/arch/xsimd_isa.hpp b/include/xsimd/arch/xsimd_isa.hpp index 061c73598..484eab49d 100644 --- a/include/xsimd/arch/xsimd_isa.hpp +++ b/include/xsimd/arch/xsimd_isa.hpp @@ -17,8 +17,10 @@ #include "./xsimd_common_fwd.hpp" // v15 API under migration -#if XSIMD_WITH_NEON || XSIMD_WITH_SVE +#if XSIMD_WITH_NEON #include "../v15/arithmetic/arm.hpp" +#elif XSIMD_WITH_SSE2 +#include "../v15/arithmetic/x86.hpp" #endif #if XSIMD_WITH_EMULATED diff --git a/include/xsimd/arch/xsimd_sse2.hpp b/include/xsimd/arch/xsimd_sse2.hpp index eccba3b36..cae526f43 100644 --- a/include/xsimd/arch/xsimd_sse2.hpp +++ b/include/xsimd/arch/xsimd_sse2.hpp @@ -14,6 +14,7 @@ #include "../types/xsimd_batch_constant.hpp" #include "../types/xsimd_sse2_register.hpp" +#include "../v15/kernel_fwd.hpp" #include "./utils/shifts.hpp" #include @@ -81,33 +82,6 @@ namespace xsimd return _mm_andnot_ps(sign_mask, self); } - // add - template >> - XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept - { - if constexpr (sizeof(T) == 1) - { - return _mm_add_epi8(self, other); - } - else if constexpr (sizeof(T) == 2) - { - return _mm_add_epi16(self, other); - } - else if constexpr (sizeof(T) == 4) - { - return _mm_add_epi32(self, other); - } - else if constexpr (sizeof(T) == 8) - { - return _mm_add_epi64(self, other); - } - else - { - assert(false && "unsupported arch/op combination"); - return {}; - } - } - template XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept { diff --git a/include/xsimd/types/xsimd_all_registers.hpp b/include/xsimd/types/xsimd_all_registers.hpp index 7dc66f39e..318c972b7 100644 --- a/include/xsimd/types/xsimd_all_registers.hpp +++ b/include/xsimd/types/xsimd_all_registers.hpp @@ -10,36 +10,12 @@ * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ -#include "./xsimd_sve_register.hpp" -#include "./xsimd_avx2_register.hpp" -#include "./xsimd_avx512bw_register.hpp" -#include "./xsimd_avx512cd_register.hpp" -#include "./xsimd_avx512dq_register.hpp" -#include "./xsimd_avx512er_register.hpp" -#include "./xsimd_avx512f_register.hpp" -#include "./xsimd_avx512ifma_register.hpp" -#include "./xsimd_avx512pf_register.hpp" -#include "./xsimd_avx512vbmi2_register.hpp" -#include "./xsimd_avx512vbmi_register.hpp" -#include "./xsimd_avx512vl_register.hpp" -#include "./xsimd_avx512vnni_avx512bw_register.hpp" -#include "./xsimd_avx512vnni_avx512vbmi2_register.hpp" -#include "./xsimd_avx_register.hpp" -#include "./xsimd_avxvnni_register.hpp" -#include "./xsimd_fma3_avx2_128_register.hpp" -#include "./xsimd_fma3_avx2_register.hpp" -#include "./xsimd_fma3_avx_register.hpp" -#include "./xsimd_fma3_sse_register.hpp" -#include "./xsimd_fma4_register.hpp" -#include "./xsimd_i8mm_neon64_register.hpp" +#include "./xsimd_arm_registers.hpp" #include "./xsimd_rvv_register.hpp" -#include "./xsimd_sse2_register.hpp" -#include "./xsimd_sse3_register.hpp" -#include "./xsimd_sse4_1_register.hpp" -#include "./xsimd_sse4_2_register.hpp" #include "./xsimd_vsx_register.hpp" #include "./xsimd_vxe_register.hpp" #include "./xsimd_wasm_register.hpp" +#include "./xsimd_x86_registers.hpp" #if XSIMD_WITH_EMULATED #include "./xsimd_emulated_register.hpp" diff --git a/include/xsimd/types/xsimd_x86_registers.hpp b/include/xsimd/types/xsimd_x86_registers.hpp new file mode 100644 index 000000000..e6b226b50 --- /dev/null +++ b/include/xsimd/types/xsimd_x86_registers.hpp @@ -0,0 +1,36 @@ +/*************************************************************************** + * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * + * Martin Renou * + * Copyright (c) QuantStack * + * Copyright (c) Serge Guelton * + * Copyright (c) Marco Barbone * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#include "./xsimd_avx2_register.hpp" +#include "./xsimd_avx512bw_register.hpp" +#include "./xsimd_avx512cd_register.hpp" +#include "./xsimd_avx512dq_register.hpp" +#include "./xsimd_avx512er_register.hpp" +#include "./xsimd_avx512f_register.hpp" +#include "./xsimd_avx512ifma_register.hpp" +#include "./xsimd_avx512pf_register.hpp" +#include "./xsimd_avx512vbmi2_register.hpp" +#include "./xsimd_avx512vbmi_register.hpp" +#include "./xsimd_avx512vl_register.hpp" +#include "./xsimd_avx512vnni_avx512bw_register.hpp" +#include "./xsimd_avx512vnni_avx512vbmi2_register.hpp" +#include "./xsimd_avx_register.hpp" +#include "./xsimd_avxvnni_register.hpp" +#include "./xsimd_fma3_avx2_128_register.hpp" +#include "./xsimd_fma3_avx2_register.hpp" +#include "./xsimd_fma3_avx_register.hpp" +#include "./xsimd_fma3_sse_register.hpp" +#include "./xsimd_fma4_register.hpp" +#include "./xsimd_sse2_register.hpp" +#include "./xsimd_sse3_register.hpp" +#include "./xsimd_sse4_1_register.hpp" +#include "./xsimd_sse4_2_register.hpp" diff --git a/include/xsimd/v15/arithmetic/x86.hpp b/include/xsimd/v15/arithmetic/x86.hpp new file mode 100644 index 000000000..e6b8800d5 --- /dev/null +++ b/include/xsimd/v15/arithmetic/x86.hpp @@ -0,0 +1,187 @@ +/**************************************************************************** + * Copyright (c) xsimd contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ARITHMETIC_X86_HPP +#define XSIMD_ARITHMETIC_X86_HPP + +#include "../../arch/utils/x86.hpp" +#include "../../config/xsimd_macros.hpp" +#include "../../types/xsimd_batch.hpp" +#include "../../utils/xsimd_type_traits.hpp" +#include "../kernel_fwd.hpp" + +#include + +namespace xsimd::kernel +{ + namespace detail + { + /// A explicit trap with template so that type and architecture show in the error. + template + constexpr void unsupported() + { + // static_assert(false) in a discarded if constexpr branch is only well-formed + // since C++23 (P2593). + static_assert(!std::is_same_v, "unsupported data type for the given x86 architecture"); + } + + template + XSIMD_INLINE batch mm_add(batch lhs, batch rhs) noexcept + { + if constexpr (std::is_same_v) + { + return _mm_add_ps(lhs, rhs); + } + else if constexpr (std::is_same_v) + { + return _mm_add_pd(lhs, rhs); + } + else if constexpr (std::is_integral_v && sizeof(T) == 1) + { + return _mm_add_epi8(lhs, rhs); + } + else if constexpr (std::is_integral_v && sizeof(T) == 2) + { + return _mm_add_epi16(lhs, rhs); + } + else if constexpr (std::is_integral_v && sizeof(T) == 4) + { + return _mm_add_epi32(lhs, rhs); + } + else if constexpr (std::is_integral_v && sizeof(T) == 8) + { + return _mm_add_epi64(lhs, rhs); + } + else + { + unsupported(); + } + } + + template + XSIMD_INLINE batch mm256_add(batch lhs, batch rhs) noexcept + { + if constexpr (std::is_same_v) + { + return _mm256_add_ps(lhs, rhs); + } + else if constexpr (std::is_same_v) + { + return _mm256_add_pd(lhs, rhs); + } + else if constexpr (std::is_integral_v && std::is_base_of_v) + { + + if constexpr (sizeof(T) == 1) + { + return _mm256_add_epi8(lhs, rhs); + } + else if constexpr (sizeof(T) == 2) + { + return _mm256_add_epi16(lhs, rhs); + } + else if constexpr (sizeof(T) == 4) + { + return _mm256_add_epi32(lhs, rhs); + } + else if constexpr (sizeof(T) == 8) + { + return _mm256_add_epi64(lhs, rhs); + } + else + { + unsupported(); + } + } + else + { + unsupported(); + } + } + + template + XSIMD_INLINE batch mm512_add(batch lhs, batch rhs) noexcept + { + if constexpr (std::is_same_v) + { + return _mm512_add_ps(lhs, rhs); + } + else if constexpr (std::is_same_v) + { + return _mm512_add_pd(lhs, rhs); + } + else if constexpr (std::is_integral_v && sizeof(T) == 8) + { + return _mm512_add_epi64(lhs, rhs); + } + else if constexpr (std::is_integral_v && sizeof(T) == 4) + { + return _mm512_add_epi32(lhs, rhs); + } + else if constexpr (std::is_integral_v && std::is_base_of_v) + { + if constexpr (sizeof(T) == 1) + { + return _mm512_add_epi8(lhs, rhs); + } + else if constexpr (sizeof(T) == 2) + { + return _mm512_add_epi16(lhs, rhs); + } + else + { + unsupported(); + } + } + else + { + unsupported(); + } + } + } + + template + XSIMD_INLINE batch add(batch lhs, batch rhs) noexcept + { + constexpr auto recurse_add = [](auto l, auto r) + { return kernel::add(l, r); }; + + if constexpr (std::is_base_of_v) + { + if constexpr (!std::is_base_of_v && std::is_integral_v && sizeof(T) <= 2) + { + return detail::apply_on_halves(recurse_add, lhs, rhs); + } + else + { + return detail::mm512_add(lhs, rhs); + } + } + else if constexpr (std::is_base_of_v) + { + if constexpr (std::is_integral_v && std::is_base_of_v) + { + return detail::mm256_add(lhs, rhs); + } + else + { + return detail::apply_on_halves(recurse_add, lhs, rhs); + } + } + else if constexpr (std::is_base_of_v) // SSE family and avx_128 + { + return detail::mm_add(lhs, rhs); + } + else + { + detail::unsupported(); + } + } +} + +#endif