[WIP][POC][BenchmarkingOnly] PFOR interleaved and fastlanes numbers

.gitignore

@@ -94,3 +94,6 @@ rat.txt
 
 # for ODBC DLL
 *.rc
+
+# Local out-of-tree benchmark build dir
+cpp/build-bench/

cpp/cmake_modules/DefineOptions.cmake

@@ -194,6 +194,9 @@ takes precedence over ccache if a storage backend is configured" ON)
 
   define_option(ARROW_GGDB_DEBUG "Pass -ggdb flag to debug builds" ON)
 
+  define_option(ARROW_RELEASE_O3
+                "Keep CMake's default -O3 in Release builds instead of -O2" OFF)
+
   define_option(ARROW_WITH_MUSL "Whether the system libc is musl or not" OFF)
 
   define_option(ARROW_ENABLE_THREADING "Enable threading in Arrow core" ON)

cpp/cmake_modules/SetupCxxFlags.cmake

@@ -632,20 +632,25 @@ endif()
 #   Same as Release, except with debug symbols enabled.
 
 if(NOT MSVC)
+  # CMake's default Release flags are "-O3 -DNDEBUG"; the appends below
+  # downgrade them to -O2, since the last -O flag wins in GCC. Set
+  # ARROW_RELEASE_O3 to skip the downgrade: the bit-packing kernels are
+  # built around the inlining and unrolling -O3 enables, and the
+  # throughput figures quoted for them were measured with it in effect.
   set(C_RELEASE_FLAGS "")
-  if(CMAKE_C_FLAGS_RELEASE MATCHES "-O3")
+  if(CMAKE_C_FLAGS_RELEASE MATCHES "-O3" AND NOT ARROW_RELEASE_O3)
     string(APPEND C_RELEASE_FLAGS " -O2")
   endif()
   set(CXX_RELEASE_FLAGS "")
-  if(CMAKE_CXX_FLAGS_RELEASE MATCHES "-O3")
+  if(CMAKE_CXX_FLAGS_RELEASE MATCHES "-O3" AND NOT ARROW_RELEASE_O3)
     string(APPEND CXX_RELEASE_FLAGS " -O2")
   endif()
   set(C_RELWITHDEBINFO_FLAGS "")
-  if(CMAKE_C_FLAGS_RELWITHDEBINFO MATCHES "-O3")
+  if(CMAKE_C_FLAGS_RELWITHDEBINFO MATCHES "-O3" AND NOT ARROW_RELEASE_O3)
     string(APPEND C_RELWITHDEBINFO_FLAGS " -O2")
   endif()
   set(CXX_RELWITHDEBINFO_FLAGS "")
-  if(CMAKE_CXX_FLAGS_RELWITHDEBINFO MATCHES "-O3")
+  if(CMAKE_CXX_FLAGS_RELWITHDEBINFO MATCHES "-O3" AND NOT ARROW_RELEASE_O3)
     string(APPEND CXX_RELWITHDEBINFO_FLAGS " -O2")
   endif()
   if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")

cpp/src/arrow/CMakeLists.txt

@@ -551,6 +551,7 @@ set(ARROW_UTIL_SRCS
     util/decimal.cc
     util/delimiting.cc
     util/dict_util.cc
+    util/fastlanes/interleaved_pfor_baseline.cc
     util/fixed_width_internal.cc
     util/float16.cc
     util/formatting.cc
@@ -566,6 +567,8 @@ set(ARROW_UTIL_SRCS
     util/math_internal.cc
     util/memory.cc
     util/mutex.cc
+    util/pfor/pfor.cc
+    util/pfor/pfor_wrapper.cc
     util/ree_util.cc
     util/secure_string.cc
     util/string.cc
@@ -591,6 +594,28 @@ append_runtime_avx512_src(ARROW_UTIL_SRCS util/bpacking_simd_avx512.cc)
 append_runtime_sve128_src(ARROW_UTIL_SRCS util/bpacking_simd_128_alt.cc)
 append_runtime_sve256_src(ARROW_UTIL_SRCS util/bpacking_simd_256.cc)
 
+# One source compiled once per instruction set, the way bpacking_simd_256.cc is
+# registered for both AVX2 and SVE256 above: the file forks on the platform
+# macros, and only one of them is ever defined on a given target.
+append_runtime_avx2_src(ARROW_UTIL_SRCS util/fastlanes/interleaved_pfor_simd.cc)
+append_runtime_sve128_src(ARROW_UTIL_SRCS util/fastlanes/interleaved_pfor_simd.cc)
+
+# The interleaved kernels are portable C++ with no intrinsics, so what they
+# compile to is decided by the optimizer, not by the source. At width 16 gcc 11.5
+# emits 289 instructions and no vector operations at -O2, 322 with 81 vector
+# operations at Release's -O2 -ftree-vectorize, and 849 with 513 at -O3; the
+# published throughput figures for this layout were all taken at the last of
+# those. APPEND rather than a plain set, because the two calls above have already
+# put the instruction-set flags in this property. Release only -- Debug and the
+# sanitizer builds want their own level, and MSVC spells this differently.
+if(NOT MSVC)
+  set_property(SOURCE util/fastlanes/interleaved_pfor_baseline.cc
+                      util/fastlanes/interleaved_pfor_simd.cc
+               APPEND
+               PROPERTY COMPILE_OPTIONS
+                        "$<$<OR:$<CONFIG:Release>,$<CONFIG:RelWithDebInfo>>:-O3>")
+endif()
+
 if(ARROW_WITH_BROTLI)
   list(APPEND ARROW_UTIL_SRCS util/compression_brotli.cc)
 endif()

cpp/src/arrow/meson.build

@@ -204,6 +204,8 @@ arrow_util_srcs = [
     'util/math_internal.cc',
     'util/memory.cc',
     'util/mutex.cc',
+    'util/pfor/pfor.cc',
+    'util/pfor/pfor_wrapper.cc',
     'util/ree_util.cc',
     'util/secure_string.cc',
     'util/string.cc',

cpp/src/arrow/util/CMakeLists.txt

@@ -118,6 +118,10 @@ add_arrow_test(threading-utility-test
                test_common.cc
                thread_pool_test.cc)
 
+add_arrow_test(pfor-test SOURCES pfor/pfor_test.cc)
+
+add_arrow_benchmark(pfor/pfor_benchmark)
+
 add_arrow_benchmark(bit_block_counter_benchmark)
 add_arrow_benchmark(bit_util_benchmark)
 add_arrow_benchmark(bitmap_reader_benchmark)

cpp/src/arrow/util/bpacking.cc

@@ -38,7 +38,29 @@ struct UnpackDynamicFunction {
         ARROW_DISPATCH_TARGET_SVE256(&bpacking::unpack_sve256<Uint>)  //
         ARROW_DISPATCH_TARGET_SSE4_2(&bpacking::unpack_sse4_2<Uint>)  //
         ARROW_DISPATCH_TARGET_AVX2(&bpacking::unpack_avx2<Uint>)      //
-        ARROW_DISPATCH_TARGET_AVX512(&bpacking::unpack_avx512<Uint>)  //
+        // Cap bit-unpack dispatch at 256 bits. The generated AVX-512 kernels
+        // assemble vectors from scalar loads and use out-of-line calls, making
+        // them slower than the AVX2 path. Re-enable this target when its kernels
+        // use vector loads directly.
+        // ARROW_DISPATCH_TARGET_AVX512(&bpacking::unpack_avx512<Uint>)  //
+    };
+  }
+};
+
+template <typename Uint>
+struct UnpackBiasDynamicFunction {
+  using FunctionType = decltype(&bpacking::unpack_bias_scalar<Uint>);
+
+  static constexpr auto targets() {
+    return std::array{
+        ARROW_DISPATCH_TARGET_NONE(&bpacking::unpack_bias_scalar<Uint>)    //
+        ARROW_DISPATCH_TARGET_NEON(&bpacking::unpack_bias_neon<Uint>)      //
+        ARROW_DISPATCH_TARGET_SVE128(&bpacking::unpack_bias_sve128<Uint>)  //
+        ARROW_DISPATCH_TARGET_SVE256(&bpacking::unpack_bias_sve256<Uint>)  //
+        ARROW_DISPATCH_TARGET_SSE4_2(&bpacking::unpack_bias_sse4_2<Uint>)  //
+        ARROW_DISPATCH_TARGET_AVX2(&bpacking::unpack_bias_avx2<Uint>)      //
+        // Capped at 256 bits for the reason given in UnpackDynamicFunction above.
+        // ARROW_DISPATCH_TARGET_AVX512(&bpacking::unpack_bias_avx512<Uint>)  //
     };
   }
 };
@@ -57,4 +79,19 @@ template void unpack<uint16_t>(const uint8_t*, uint16_t*, const UnpackOptions&);
 template void unpack<uint32_t>(const uint8_t*, uint32_t*, const UnpackOptions&);
 template void unpack<uint64_t>(const uint8_t*, uint64_t*, const UnpackOptions&);
 
+template <typename Uint>
+void unpack_bias(const uint8_t* in, Uint* out, const UnpackOptions& opts, Uint bias) {
+  static const DynamicDispatch<UnpackBiasDynamicFunction<Uint>> dispatch;
+  return dispatch(in, out, opts, bias);
+}
+
+template void unpack_bias<uint8_t>(const uint8_t*, uint8_t*, const UnpackOptions&,
+                                   uint8_t);
+template void unpack_bias<uint16_t>(const uint8_t*, uint16_t*, const UnpackOptions&,
+                                    uint16_t);
+template void unpack_bias<uint32_t>(const uint8_t*, uint32_t*, const UnpackOptions&,
+                                    uint32_t);
+template void unpack_bias<uint64_t>(const uint8_t*, uint64_t*, const UnpackOptions&,
+                                    uint64_t);
+
 }  // namespace arrow::internal