Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -94,3 +94,6 @@ rat.txt

# for ODBC DLL
*.rc

# Local out-of-tree benchmark build dir
cpp/build-bench/
2 changes: 2 additions & 0 deletions cpp/src/arrow/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,8 @@ set(ARROW_UTIL_SRCS
util/math_internal.cc
util/memory.cc
util/mutex.cc
util/pfor/pfor.cc
util/pfor/pfor_wrapper.cc
util/ree_util.cc
util/secure_string.cc
util/string.cc
Expand Down
2 changes: 2 additions & 0 deletions cpp/src/arrow/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,8 @@ arrow_util_srcs = [
'util/math_internal.cc',
'util/memory.cc',
'util/mutex.cc',
'util/pfor/pfor.cc',
'util/pfor/pfor_wrapper.cc',
'util/ree_util.cc',
'util/secure_string.cc',
'util/string.cc',
Expand Down
4 changes: 4 additions & 0 deletions cpp/src/arrow/util/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,10 @@ add_arrow_test(threading-utility-test
test_common.cc
thread_pool_test.cc)

add_arrow_test(pfor-test SOURCES pfor/pfor_test.cc)

add_arrow_benchmark(pfor/pfor_benchmark)

add_arrow_benchmark(bit_block_counter_benchmark)
add_arrow_benchmark(bit_util_benchmark)
add_arrow_benchmark(bitmap_reader_benchmark)
Expand Down
32 changes: 32 additions & 0 deletions cpp/src/arrow/util/bpacking.cc
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,23 @@ struct UnpackDynamicFunction {
}
};

template <typename Uint>
struct UnpackBiasDynamicFunction {
using FunctionType = decltype(&bpacking::unpack_bias_scalar<Uint>);

static constexpr auto targets() {
return std::array{
ARROW_DISPATCH_TARGET_NONE(&bpacking::unpack_bias_scalar<Uint>) //
ARROW_DISPATCH_TARGET_NEON(&bpacking::unpack_bias_neon<Uint>) //
ARROW_DISPATCH_TARGET_SVE128(&bpacking::unpack_bias_sve128<Uint>) //
ARROW_DISPATCH_TARGET_SVE256(&bpacking::unpack_bias_sve256<Uint>) //
ARROW_DISPATCH_TARGET_SSE4_2(&bpacking::unpack_bias_sse4_2<Uint>) //
ARROW_DISPATCH_TARGET_AVX2(&bpacking::unpack_bias_avx2<Uint>) //
ARROW_DISPATCH_TARGET_AVX512(&bpacking::unpack_bias_avx512<Uint>) //
};
}
};

} // namespace

template <typename Uint>
Expand All @@ -57,4 +74,19 @@ template void unpack<uint16_t>(const uint8_t*, uint16_t*, const UnpackOptions&);
template void unpack<uint32_t>(const uint8_t*, uint32_t*, const UnpackOptions&);
template void unpack<uint64_t>(const uint8_t*, uint64_t*, const UnpackOptions&);

template <typename Uint>
void unpack_bias(const uint8_t* in, Uint* out, const UnpackOptions& opts, Uint bias) {
static const DynamicDispatch<UnpackBiasDynamicFunction<Uint>> dispatch;
return dispatch(in, out, opts, bias);
}

template void unpack_bias<uint8_t>(const uint8_t*, uint8_t*, const UnpackOptions&,
uint8_t);
template void unpack_bias<uint16_t>(const uint8_t*, uint16_t*, const UnpackOptions&,
uint16_t);
template void unpack_bias<uint32_t>(const uint8_t*, uint32_t*, const UnpackOptions&,
uint32_t);
template void unpack_bias<uint64_t>(const uint8_t*, uint64_t*, const UnpackOptions&,
uint64_t);

} // namespace arrow::internal
602 changes: 330 additions & 272 deletions cpp/src/arrow/util/bpacking_dispatch_internal.h

Large diffs are not rendered by default.

25 changes: 25 additions & 0 deletions cpp/src/arrow/util/bpacking_internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,29 @@ extern template ARROW_TEMPLATE_EXPORT void unpack<uint32_t>( //
extern template ARROW_TEMPLATE_EXPORT void unpack<uint64_t>( //
const uint8_t* in, uint64_t* out, const UnpackOptions& opts);

/// \brief Unpack, adding `bias` to every value.
///
/// Equivalent to `unpack()` followed by a loop adding `bias`, but the addition
/// happens inside the unpacker before its store, so the output is traversed
/// once instead of twice. Frame-of-reference decoders are the motivating
/// caller: the separate add pass costs as much as the unpack it follows.
///
/// `bias + value` is computed in `Uint`, so it wraps rather than saturating.
/// Not defined for `bool` output, where an add has no meaning.
template <typename Uint>
ARROW_EXPORT void unpack_bias(const uint8_t* in, Uint* out, const UnpackOptions& opts,
Uint bias);

extern template ARROW_TEMPLATE_EXPORT void unpack_bias<uint8_t>( //
const uint8_t* in, uint8_t* out, const UnpackOptions& opts, uint8_t bias);

extern template ARROW_TEMPLATE_EXPORT void unpack_bias<uint16_t>( //
const uint8_t* in, uint16_t* out, const UnpackOptions& opts, uint16_t bias);

extern template ARROW_TEMPLATE_EXPORT void unpack_bias<uint32_t>( //
const uint8_t* in, uint32_t* out, const UnpackOptions& opts, uint32_t bias);

extern template ARROW_TEMPLATE_EXPORT void unpack_bias<uint64_t>( //
const uint8_t* in, uint64_t* out, const UnpackOptions& opts, uint64_t bias);

} // namespace arrow::internal
15 changes: 15 additions & 0 deletions cpp/src/arrow/util/bpacking_scalar.cc
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,19 @@ template void unpack_scalar<uint16_t>(const uint8_t*, uint16_t*, const UnpackOpt
template void unpack_scalar<uint32_t>(const uint8_t*, uint32_t*, const UnpackOptions&);
template void unpack_scalar<uint64_t>(const uint8_t*, uint64_t*, const UnpackOptions&);

template <typename Uint>
void unpack_bias_scalar(const uint8_t* in, Uint* out, const UnpackOptions& opts,
Uint bias) {
return unpack_jump<ScalarUnpackerForWidth, /*kHasBias=*/true>(in, out, opts, bias);
}

template void unpack_bias_scalar<uint8_t>(const uint8_t*, uint8_t*, const UnpackOptions&,
uint8_t);
template void unpack_bias_scalar<uint16_t>(const uint8_t*, uint16_t*,
const UnpackOptions&, uint16_t);
template void unpack_bias_scalar<uint32_t>(const uint8_t*, uint32_t*,
const UnpackOptions&, uint32_t);
template void unpack_bias_scalar<uint64_t>(const uint8_t*, uint64_t*,
const UnpackOptions&, uint64_t);

} // namespace arrow::internal::bpacking
16 changes: 16 additions & 0 deletions cpp/src/arrow/util/bpacking_scalar_internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,20 @@ extern template ARROW_TEMPLATE_EXPORT void unpack_scalar<uint32_t>(
extern template ARROW_TEMPLATE_EXPORT void unpack_scalar<uint64_t>(
const uint8_t* in, uint64_t* out, const UnpackOptions& opts);

template <typename Uint>
ARROW_EXPORT void unpack_bias_scalar(const uint8_t* in, Uint* out,
const UnpackOptions& opts, Uint bias);

extern template ARROW_TEMPLATE_EXPORT void unpack_bias_scalar<uint8_t>(
const uint8_t* in, uint8_t* out, const UnpackOptions& opts, uint8_t bias);

extern template ARROW_TEMPLATE_EXPORT void unpack_bias_scalar<uint16_t>(
const uint8_t* in, uint16_t* out, const UnpackOptions& opts, uint16_t bias);

extern template ARROW_TEMPLATE_EXPORT void unpack_bias_scalar<uint32_t>(
const uint8_t* in, uint32_t* out, const UnpackOptions& opts, uint32_t bias);

extern template ARROW_TEMPLATE_EXPORT void unpack_bias_scalar<uint64_t>(
const uint8_t* in, uint64_t* out, const UnpackOptions& opts, uint64_t bias);

} // namespace arrow::internal::bpacking
18 changes: 18 additions & 0 deletions cpp/src/arrow/util/bpacking_simd_128.cc
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@

#if defined(ARROW_HAVE_NEON)
# define UNPACK_PLATFORM unpack_neon
# define UNPACK_BIAS_PLATFORM unpack_bias_neon
# define KERNEL_PLATFORM KernelNeon
#elif defined(ARROW_HAVE_SSE4_2)
# define UNPACK_PLATFORM unpack_sse4_2
# define UNPACK_BIAS_PLATFORM unpack_bias_sse4_2
# define KERNEL_PLATFORM KernelSse42
#endif

Expand All @@ -45,7 +47,23 @@ template void UNPACK_PLATFORM<uint16_t>(const uint8_t*, uint16_t*, const UnpackO
template void UNPACK_PLATFORM<uint32_t>(const uint8_t*, uint32_t*, const UnpackOptions&);
template void UNPACK_PLATFORM<uint64_t>(const uint8_t*, uint64_t*, const UnpackOptions&);

template <typename Uint>
void UNPACK_BIAS_PLATFORM(const uint8_t* in, Uint* out, const UnpackOptions& opts,
Uint bias) {
return unpack_jump<KERNEL_PLATFORM, /*kHasBias=*/true>(in, out, opts, bias);
}

template void UNPACK_BIAS_PLATFORM<uint8_t>(const uint8_t*, uint8_t*,
const UnpackOptions&, uint8_t);
template void UNPACK_BIAS_PLATFORM<uint16_t>(const uint8_t*, uint16_t*,
const UnpackOptions&, uint16_t);
template void UNPACK_BIAS_PLATFORM<uint32_t>(const uint8_t*, uint32_t*,
const UnpackOptions&, uint32_t);
template void UNPACK_BIAS_PLATFORM<uint64_t>(const uint8_t*, uint64_t*,
const UnpackOptions&, uint64_t);

} // namespace arrow::internal::bpacking

# undef UNPACK_PLATFORM
# undef UNPACK_BIAS_PLATFORM
#endif // UNPACK_PLATFORM
17 changes: 17 additions & 0 deletions cpp/src/arrow/util/bpacking_simd_128_alt.cc
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

#if defined(ARROW_HAVE_RUNTIME_SVE128)
# define UNPACK_PLATFORM unpack_sve128
# define UNPACK_BIAS_PLATFORM unpack_bias_sve128
# define KERNEL_PLATFORM KernelSve128
#endif

Expand Down Expand Up @@ -46,6 +47,22 @@ template void UNPACK_PLATFORM<uint16_t>(const uint8_t*, uint16_t*, const UnpackO
template void UNPACK_PLATFORM<uint32_t>(const uint8_t*, uint32_t*, const UnpackOptions&);
template void UNPACK_PLATFORM<uint64_t>(const uint8_t*, uint64_t*, const UnpackOptions&);

template <typename Uint>
void UNPACK_BIAS_PLATFORM(const uint8_t* in, Uint* out, const UnpackOptions& opts,
Uint bias) {
return unpack_jump<KERNEL_PLATFORM, /*kHasBias=*/true>(in, out, opts, bias);
}

template void UNPACK_BIAS_PLATFORM<uint8_t>(const uint8_t*, uint8_t*,
const UnpackOptions&, uint8_t);
template void UNPACK_BIAS_PLATFORM<uint16_t>(const uint8_t*, uint16_t*,
const UnpackOptions&, uint16_t);
template void UNPACK_BIAS_PLATFORM<uint32_t>(const uint8_t*, uint32_t*,
const UnpackOptions&, uint32_t);
template void UNPACK_BIAS_PLATFORM<uint64_t>(const uint8_t*, uint64_t*,
const UnpackOptions&, uint64_t);

} // namespace arrow::internal::bpacking

#undef UNPACK_PLATFORM
#undef UNPACK_BIAS_PLATFORM
18 changes: 18 additions & 0 deletions cpp/src/arrow/util/bpacking_simd_256.cc
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@

#if defined(ARROW_HAVE_SVE256) || defined(ARROW_HAVE_RUNTIME_SVE256)
# define UNPACK_PLATFORM unpack_sve256
# define UNPACK_BIAS_PLATFORM unpack_bias_sve256
# define KERNEL_PLATFORM KernelSve256
#elif defined(ARROW_HAVE_RUNTIME_AVX2)
# define UNPACK_PLATFORM unpack_avx2
# define UNPACK_BIAS_PLATFORM unpack_bias_avx2
# define KERNEL_PLATFORM KernelAvx2
#endif

Expand Down Expand Up @@ -48,6 +50,22 @@ template void UNPACK_PLATFORM<uint16_t>(const uint8_t*, uint16_t*, const UnpackO
template void UNPACK_PLATFORM<uint32_t>(const uint8_t*, uint32_t*, const UnpackOptions&);
template void UNPACK_PLATFORM<uint64_t>(const uint8_t*, uint64_t*, const UnpackOptions&);

template <typename Uint>
void UNPACK_BIAS_PLATFORM(const uint8_t* in, Uint* out, const UnpackOptions& opts,
Uint bias) {
return unpack_jump<KERNEL_PLATFORM, /*kHasBias=*/true>(in, out, opts, bias);
}

template void UNPACK_BIAS_PLATFORM<uint8_t>(const uint8_t*, uint8_t*,
const UnpackOptions&, uint8_t);
template void UNPACK_BIAS_PLATFORM<uint16_t>(const uint8_t*, uint16_t*,
const UnpackOptions&, uint16_t);
template void UNPACK_BIAS_PLATFORM<uint32_t>(const uint8_t*, uint32_t*,
const UnpackOptions&, uint32_t);
template void UNPACK_BIAS_PLATFORM<uint64_t>(const uint8_t*, uint64_t*,
const UnpackOptions&, uint64_t);

} // namespace arrow::internal::bpacking

#undef UNPACK_PLATFORM
#undef UNPACK_BIAS_PLATFORM
15 changes: 15 additions & 0 deletions cpp/src/arrow/util/bpacking_simd_avx512.cc
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,19 @@ template void unpack_avx512<uint16_t>(const uint8_t*, uint16_t*, const UnpackOpt
template void unpack_avx512<uint32_t>(const uint8_t*, uint32_t*, const UnpackOptions&);
template void unpack_avx512<uint64_t>(const uint8_t*, uint64_t*, const UnpackOptions&);

template <typename Uint>
void unpack_bias_avx512(const uint8_t* in, Uint* out, const UnpackOptions& opts,
Uint bias) {
return unpack_jump<Simd512UnpackerForWidth, /*kHasBias=*/true>(in, out, opts, bias);
}

template void unpack_bias_avx512<uint8_t>(const uint8_t*, uint8_t*, const UnpackOptions&,
uint8_t);
template void unpack_bias_avx512<uint16_t>(const uint8_t*, uint16_t*,
const UnpackOptions&, uint16_t);
template void unpack_bias_avx512<uint32_t>(const uint8_t*, uint32_t*,
const UnpackOptions&, uint32_t);
template void unpack_bias_avx512<uint64_t>(const uint8_t*, uint64_t*,
const UnpackOptions&, uint64_t);

} // namespace arrow::internal::bpacking
72 changes: 72 additions & 0 deletions cpp/src/arrow/util/bpacking_simd_internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,10 @@ namespace arrow::internal::bpacking {

#if defined(ARROW_HAVE_NEON)
# define UNPACK_ARCH128 unpack_neon
# define UNPACK_BIAS_ARCH128 unpack_bias_neon
#elif defined(ARROW_HAVE_SSE4_2)
# define UNPACK_ARCH128 unpack_sse4_2
# define UNPACK_BIAS_ARCH128 unpack_bias_sse4_2
#endif

#if defined(UNPACK_ARCH128)
Expand All @@ -50,11 +52,29 @@ extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128<uint32_t>(
extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128<uint64_t>(
const uint8_t* in, uint64_t* out, const UnpackOptions& opts);

template <typename Uint>
ARROW_EXPORT void UNPACK_BIAS_ARCH128(const uint8_t* in, Uint* out,
const UnpackOptions& opts, Uint bias);

extern template ARROW_TEMPLATE_EXPORT void UNPACK_BIAS_ARCH128<uint8_t>(
const uint8_t* in, uint8_t* out, const UnpackOptions& opts, uint8_t bias);

extern template ARROW_TEMPLATE_EXPORT void UNPACK_BIAS_ARCH128<uint16_t>(
const uint8_t* in, uint16_t* out, const UnpackOptions& opts, uint16_t bias);

extern template ARROW_TEMPLATE_EXPORT void UNPACK_BIAS_ARCH128<uint32_t>(
const uint8_t* in, uint32_t* out, const UnpackOptions& opts, uint32_t bias);

extern template ARROW_TEMPLATE_EXPORT void UNPACK_BIAS_ARCH128<uint64_t>(
const uint8_t* in, uint64_t* out, const UnpackOptions& opts, uint64_t bias);

#endif // UNPACK_ARCH128
#undef UNPACK_ARCH128
#undef UNPACK_BIAS_ARCH128

#if defined(ARROW_HAVE_RUNTIME_SVE128)
# define UNPACK_ARCH128_ALT unpack_sve128
# define UNPACK_BIAS_ARCH128_ALT unpack_bias_sve128
#endif

#if defined(UNPACK_ARCH128_ALT)
Expand All @@ -78,13 +98,32 @@ extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128_ALT<uint32_t>(
extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128_ALT<uint64_t>(
const uint8_t* in, uint64_t* out, const UnpackOptions& opts);

template <typename Uint>
ARROW_EXPORT void UNPACK_BIAS_ARCH128_ALT(const uint8_t* in, Uint* out,
const UnpackOptions& opts, Uint bias);

extern template ARROW_TEMPLATE_EXPORT void UNPACK_BIAS_ARCH128_ALT<uint8_t>(
const uint8_t* in, uint8_t* out, const UnpackOptions& opts, uint8_t bias);

extern template ARROW_TEMPLATE_EXPORT void UNPACK_BIAS_ARCH128_ALT<uint16_t>(
const uint8_t* in, uint16_t* out, const UnpackOptions& opts, uint16_t bias);

extern template ARROW_TEMPLATE_EXPORT void UNPACK_BIAS_ARCH128_ALT<uint32_t>(
const uint8_t* in, uint32_t* out, const UnpackOptions& opts, uint32_t bias);

extern template ARROW_TEMPLATE_EXPORT void UNPACK_BIAS_ARCH128_ALT<uint64_t>(
const uint8_t* in, uint64_t* out, const UnpackOptions& opts, uint64_t bias);

#endif // UNPACK_ARCH128_ALT
#undef UNPACK_ARCH128_ALT
#undef UNPACK_BIAS_ARCH128_ALT

#if defined(ARROW_HAVE_SVE256) || defined(ARROW_HAVE_RUNTIME_SVE256)
# define UNPACK_ARCH256 unpack_sve256
# define UNPACK_BIAS_ARCH256 unpack_bias_sve256
#elif defined(UNPACK_ARCH256) || defined(ARROW_HAVE_RUNTIME_AVX2)
# define UNPACK_ARCH256 unpack_avx2
# define UNPACK_BIAS_ARCH256 unpack_bias_avx2
#endif

#if defined(UNPACK_ARCH256)
Expand All @@ -107,8 +146,25 @@ extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH256<uint32_t>(
extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH256<uint64_t>(
const uint8_t* in, uint64_t* out, const UnpackOptions& opts);

template <typename Uint>
ARROW_EXPORT void UNPACK_BIAS_ARCH256(const uint8_t* in, Uint* out,
const UnpackOptions& opts, Uint bias);

extern template ARROW_TEMPLATE_EXPORT void UNPACK_BIAS_ARCH256<uint8_t>(
const uint8_t* in, uint8_t* out, const UnpackOptions& opts, uint8_t bias);

extern template ARROW_TEMPLATE_EXPORT void UNPACK_BIAS_ARCH256<uint16_t>(
const uint8_t* in, uint16_t* out, const UnpackOptions& opts, uint16_t bias);

extern template ARROW_TEMPLATE_EXPORT void UNPACK_BIAS_ARCH256<uint32_t>(
const uint8_t* in, uint32_t* out, const UnpackOptions& opts, uint32_t bias);

extern template ARROW_TEMPLATE_EXPORT void UNPACK_BIAS_ARCH256<uint64_t>(
const uint8_t* in, uint64_t* out, const UnpackOptions& opts, uint64_t bias);

#endif // UNPACK_ARCH256
#undef UNPACK_ARCH256
#undef UNPACK_BIAS_ARCH256

#if defined(ARROW_HAVE_AVX512) || defined(ARROW_HAVE_RUNTIME_AVX512)

Expand All @@ -130,6 +186,22 @@ extern template ARROW_TEMPLATE_EXPORT void unpack_avx512<uint32_t>(
extern template ARROW_TEMPLATE_EXPORT void unpack_avx512<uint64_t>(
const uint8_t* in, uint64_t* out, const UnpackOptions& opts);

template <typename Uint>
ARROW_EXPORT void unpack_bias_avx512(const uint8_t* in, Uint* out,
const UnpackOptions& opts, Uint bias);

extern template ARROW_TEMPLATE_EXPORT void unpack_bias_avx512<uint8_t>(
const uint8_t* in, uint8_t* out, const UnpackOptions& opts, uint8_t bias);

extern template ARROW_TEMPLATE_EXPORT void unpack_bias_avx512<uint16_t>(
const uint8_t* in, uint16_t* out, const UnpackOptions& opts, uint16_t bias);

extern template ARROW_TEMPLATE_EXPORT void unpack_bias_avx512<uint32_t>(
const uint8_t* in, uint32_t* out, const UnpackOptions& opts, uint32_t bias);

extern template ARROW_TEMPLATE_EXPORT void unpack_bias_avx512<uint64_t>(
const uint8_t* in, uint64_t* out, const UnpackOptions& opts, uint64_t bias);

#endif

} // namespace arrow::internal::bpacking
Loading
Loading