Add bpacking Sve 256 dynamic dispatch

Author: AntoinePrv

cpp/src/arrow/CMakeLists.txt

@@ -519,7 +519,7 @@ set(ARROW_UTIL_SRCS
     util/bitmap_ops.cc
     util/bpacking.cc
     util/bpacking_scalar.cc
-    util/bpacking_simd_default.cc
+    util/bpacking_simd_128.cc
     util/byte_size.cc
     util/byte_stream_split_internal.cc
     util/cancel.cc
@@ -564,9 +564,11 @@ set(ARROW_UTIL_SRCS
 
 append_runtime_avx2_src(ARROW_UTIL_SRCS util/byte_stream_split_internal_avx2.cc)
 
-append_runtime_avx2_src(ARROW_UTIL_SRCS util/bpacking_simd_avx2.cc)
+append_runtime_avx2_src(ARROW_UTIL_SRCS util/bpacking_simd_256.cc)
 append_runtime_avx512_src(ARROW_UTIL_SRCS util/bpacking_simd_avx512.cc)
 
+append_runtime_sve256_src(ARROW_UTIL_SRCS util/bpacking_simd_256.cc)
+
 if(ARROW_WITH_BROTLI)
   list(APPEND ARROW_UTIL_SRCS util/compression_brotli.cc)
 endif()

cpp/src/arrow/util/bpacking.cc

@@ -33,16 +33,26 @@ struct UnpackDynamicFunction {
 
   static constexpr auto implementations() {
     return std::array{
+    // x86 implementations
 #if defined(ARROW_HAVE_SSE4_2)
         Implementation{DispatchLevel::NONE, &bpacking::unpack_sse4_2<Uint>},
-#else
-        Implementation{DispatchLevel::NONE, &bpacking::unpack_scalar<Uint>},
-#endif
-#if defined(ARROW_HAVE_RUNTIME_AVX2)
+#  if defined(ARROW_HAVE_RUNTIME_AVX2)
         Implementation{DispatchLevel::AVX2, &bpacking::unpack_avx2<Uint>},
-#endif
-#if defined(ARROW_HAVE_RUNTIME_AVX512)
+#  endif
+#  if defined(ARROW_HAVE_RUNTIME_AVX512)
         Implementation{DispatchLevel::AVX512, &bpacking::unpack_avx512<Uint>},
+#  endif
+
+    // ARM implementations
+#elif defined(ARROW_HAVE_NEON)
+        Implementation{DispatchLevel::NONE, &bpacking::unpack_neon<Uint>},
+#  if defined(ARROW_HAVE_RUNTIME_SVE256)
+        Implementation{DispatchLevel::SVE256, &bpacking::unpack_sve256<Uint>},
+#  endif
+
+    // Other implementations
+#else
+        Implementation{DispatchLevel::NONE, &bpacking::unpack_scalar<Uint>},
 #endif
     };
   }
@@ -52,12 +62,14 @@ struct UnpackDynamicFunction {
 
 template <typename Uint>
 void unpack(const uint8_t* in, Uint* out, const UnpackOptions& opts) {
-#if defined(ARROW_HAVE_NEON)
-  return bpacking::unpack_neon(in, out, opts);
-#else
-  static DynamicDispatch<UnpackDynamicFunction<Uint> > dispatch;
-  return dispatch.func(in, out, opts);
-#endif
+  auto constexpr kImplementations = UnpackDynamicFunction<Uint>::implementations();
+  if constexpr (kImplementations.size() == 1) {
+    constexpr auto func = kImplementations.front().second;
+    func(in, out, opts);
+  } else {
+    static DynamicDispatch<UnpackDynamicFunction<Uint> > dispatch;
+    return dispatch.func(in, out, opts);
+  }
 }
 
 template void unpack<bool>(const uint8_t*, bool*, const UnpackOptions&);

cpp/src/arrow/util/bpacking_benchmark.cc

@@ -26,7 +26,7 @@
 #include "arrow/util/bpacking_scalar_internal.h"
 #include "arrow/util/bpacking_simd_internal.h"
 
-#if defined(ARROW_HAVE_RUNTIME_AVX2)
+#if defined(ARROW_HAVE_RUNTIME_AVX2) || defined(ARROW_HAVE_RUNTIME_SVE256)
 #  include "arrow/util/cpu_info.h"
 #endif
 
@@ -254,6 +254,33 @@ BENCHMARK_CAPTURE(BM_UnpackUint64, NeonUnaligned, false, &bpacking::unpack_neon<
     ->ArgsProduct(kBitWidthsNumValues64);
 #endif
 
+#if defined(ARROW_HAVE_RUNTIME_SVE256)
+BENCHMARK_CAPTURE(BM_UnpackBool, Sve256Unaligned, false, &bpacking::unpack_sve256<bool>,
+                  !CpuInfo::GetInstance()->IsSupported(CpuInfo::SVE256),
+                  "Sve256 not available")
+    ->ArgsProduct(kBitWidthsNumValuesBool);
+BENCHMARK_CAPTURE(BM_UnpackUint8, Sve256Unaligned, false,
+                  &bpacking::unpack_sve256<uint8_t>,
+                  !CpuInfo::GetInstance()->IsSupported(CpuInfo::SVE256),
+                  "Sve256 not available")
+    ->ArgsProduct(kBitWidthsNumValues8);
+BENCHMARK_CAPTURE(BM_UnpackUint16, Sve256Unaligned, false,
+                  &bpacking::unpack_sve256<uint16_t>,
+                  !CpuInfo::GetInstance()->IsSupported(CpuInfo::SVE256),
+                  "Sve256 not available")
+    ->ArgsProduct(kBitWidthsNumValues16);
+BENCHMARK_CAPTURE(BM_UnpackUint32, Sve256Unaligned, false,
+                  &bpacking::unpack_sve256<uint32_t>,
+                  !CpuInfo::GetInstance()->IsSupported(CpuInfo::SVE256),
+                  "Sve256 not available")
+    ->ArgsProduct(kBitWidthsNumValues32);
+BENCHMARK_CAPTURE(BM_UnpackUint64, Sve256Unaligned, false,
+                  &bpacking::unpack_sve256<uint64_t>,
+                  !CpuInfo::GetInstance()->IsSupported(CpuInfo::SVE256),
+                  "Sve256 not available")
+    ->ArgsProduct(kBitWidthsNumValues64);
+#endif
+
 BENCHMARK_CAPTURE(BM_UnpackBool, DynamicAligned, true, &unpack<bool>)
     ->ArgsProduct(kBitWidthsNumValuesBool);
 BENCHMARK_CAPTURE(BM_UnpackBool, DynamicUnaligned, false, &unpack<bool>)

…/src/arrow/util/bpacking_simd_default.cc cpp/src/arrow/util/bpacking_simd_128.cccpp/src/arrow/util/bpacking_simd_default.cc renamed to cpp/src/arrow/util/bpacking_simd_128.cc cpp/src/arrow/util/bpacking_simd_default.cc renamed to cpp/src/arrow/util/bpacking_simd_128.cc

@@ -15,25 +15,36 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/bpacking_dispatch_internal.h"
-#include "arrow/util/bpacking_internal.h"
-#include "arrow/util/bpacking_simd_internal.h"
-#include "arrow/util/bpacking_simd_kernel_internal.h"
+#if defined(ARROW_HAVE_SVE256) || defined(ARROW_HAVE_RUNTIME_SVE256)
+#  define UNPACK_PLATFORM unpack_sve256
+#elif defined(ARROW_HAVE_RUNTIME_AVX2)
+#  define UNPACK_PLATFORM unpack_avx2
+#endif
+
+#if defined(UNPACK_PLATFORM)
+
+#  include "arrow/util/bpacking_dispatch_internal.h"
+#  include "arrow/util/bpacking_internal.h"
+#  include "arrow/util/bpacking_simd_internal.h"
+#  include "arrow/util/bpacking_simd_kernel_internal.h"
 
 namespace arrow::internal::bpacking {
 
 template <typename UnpackedUint, int kPackedBitSize>
 using Simd256Kernel = Kernel<UnpackedUint, kPackedBitSize, 256>;
 
 template <typename Uint>
-void unpack_avx2(const uint8_t* in, Uint* out, const UnpackOptions& opts) {
+void UNPACK_PLATFORM(const uint8_t* in, Uint* out, const UnpackOptions& opts) {
   return unpack_jump<Simd256Kernel>(in, out, opts);
 }
 
-template void unpack_avx2<bool>(const uint8_t*, bool*, const UnpackOptions&);
-template void unpack_avx2<uint8_t>(const uint8_t*, uint8_t*, const UnpackOptions&);
-template void unpack_avx2<uint16_t>(const uint8_t*, uint16_t*, const UnpackOptions&);
-template void unpack_avx2<uint32_t>(const uint8_t*, uint32_t*, const UnpackOptions&);
-template void unpack_avx2<uint64_t>(const uint8_t*, uint64_t*, const UnpackOptions&);
+template void UNPACK_PLATFORM<bool>(const uint8_t*, bool*, const UnpackOptions&);
+template void UNPACK_PLATFORM<uint8_t>(const uint8_t*, uint8_t*, const UnpackOptions&);
+template void UNPACK_PLATFORM<uint16_t>(const uint8_t*, uint16_t*, const UnpackOptions&);
+template void UNPACK_PLATFORM<uint32_t>(const uint8_t*, uint32_t*, const UnpackOptions&);
+template void UNPACK_PLATFORM<uint64_t>(const uint8_t*, uint64_t*, const UnpackOptions&);
 
 }  // namespace arrow::internal::bpacking
+
+#  undef UNPACK_PLATFORM
+#endif  // UNPACK_PLATFORM

cpp/src/arrow/util/bpacking_simd_avx2.cc cpp/src/arrow/util/bpacking_simd_256.cccpp/src/arrow/util/bpacking_simd_avx2.cc renamed to cpp/src/arrow/util/bpacking_simd_256.cc

@@ -25,68 +25,62 @@
 namespace arrow::internal::bpacking {
 
 #if defined(ARROW_HAVE_NEON)
-
-template <typename Uint>
-ARROW_EXPORT void unpack_neon(const uint8_t* in, Uint* out, const UnpackOptions& opts);
-
-extern template ARROW_TEMPLATE_EXPORT void unpack_neon<bool>(  //
-    const uint8_t* in, bool* out, const UnpackOptions& opts);
-
-extern template ARROW_TEMPLATE_EXPORT void unpack_neon<uint8_t>(
-    const uint8_t* in, uint8_t* out, const UnpackOptions& opts);
-
-extern template ARROW_TEMPLATE_EXPORT void unpack_neon<uint16_t>(
-    const uint8_t* in, uint16_t* out, const UnpackOptions& opts);
-
-extern template ARROW_TEMPLATE_EXPORT void unpack_neon<uint32_t>(
-    const uint8_t* in, uint32_t* out, const UnpackOptions& opts);
-
-extern template ARROW_TEMPLATE_EXPORT void unpack_neon<uint64_t>(
-    const uint8_t* in, uint64_t* out, const UnpackOptions& opts);
-
+#  define UNPACK_ARCH128 unpack_neon
 #elif defined(ARROW_HAVE_SSE4_2)
+#  define UNPACK_ARCH128 unpack_sse4_2
+#endif
+
+#if defined(UNPACK_ARCH128)
 
 template <typename Uint>
-ARROW_EXPORT void unpack_sse4_2(const uint8_t* in, Uint* out, const UnpackOptions& opts);
+ARROW_EXPORT void UNPACK_ARCH128(const uint8_t* in, Uint* out, const UnpackOptions& opts);
 
-extern template ARROW_TEMPLATE_EXPORT void unpack_sse4_2<bool>(  //
+extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128<bool>(  //
     const uint8_t* in, bool* out, const UnpackOptions& opts);
 
-extern template ARROW_TEMPLATE_EXPORT void unpack_sse4_2<uint8_t>(
+extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128<uint8_t>(
     const uint8_t* in, uint8_t* out, const UnpackOptions& opts);
 
-extern template ARROW_TEMPLATE_EXPORT void unpack_sse4_2<uint16_t>(
+extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128<uint16_t>(
     const uint8_t* in, uint16_t* out, const UnpackOptions& opts);
 
-extern template ARROW_TEMPLATE_EXPORT void unpack_sse4_2<uint32_t>(
+extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128<uint32_t>(
     const uint8_t* in, uint32_t* out, const UnpackOptions& opts);
 
-extern template ARROW_TEMPLATE_EXPORT void unpack_sse4_2<uint64_t>(
+extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128<uint64_t>(
     const uint8_t* in, uint64_t* out, const UnpackOptions& opts);
 
+#endif  // UNPACK_ARCH128
+#undef UNPACK_ARCH128
+
+#if defined(ARROW_HAVE_SVE256) || defined(ARROW_HAVE_RUNTIME_SVE256)
+#  define UNPACK_ARCH256 unpack_sve256
+#elif defined(UNPACK_ARCH256) || defined(ARROW_HAVE_RUNTIME_AVX2)
+#  define UNPACK_ARCH256 unpack_avx2
 #endif
 
-#if defined(ARROW_HAVE_AVX2) || defined(ARROW_HAVE_RUNTIME_AVX2)
+#if defined(UNPACK_ARCH256)
 
 template <typename Uint>
-ARROW_EXPORT void unpack_avx2(const uint8_t* in, Uint* out, const UnpackOptions& opts);
+ARROW_EXPORT void UNPACK_ARCH256(const uint8_t* in, Uint* out, const UnpackOptions& opts);
 
-extern template ARROW_TEMPLATE_EXPORT void unpack_avx2<bool>(  //
+extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH256<bool>(  //
     const uint8_t* in, bool* out, const UnpackOptions& opts);
 
-extern template ARROW_TEMPLATE_EXPORT void unpack_avx2<uint8_t>(
+extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH256<uint8_t>(
     const uint8_t* in, uint8_t* out, const UnpackOptions& opts);
 
-extern template ARROW_TEMPLATE_EXPORT void unpack_avx2<uint16_t>(
+extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH256<uint16_t>(
     const uint8_t* in, uint16_t* out, const UnpackOptions& opts);
 
-extern template ARROW_TEMPLATE_EXPORT void unpack_avx2<uint32_t>(
+extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH256<uint32_t>(
     const uint8_t* in, uint32_t* out, const UnpackOptions& opts);
 
-extern template ARROW_TEMPLATE_EXPORT void unpack_avx2<uint64_t>(
+extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH256<uint64_t>(
     const uint8_t* in, uint64_t* out, const UnpackOptions& opts);
 
-#endif
+#endif  // UNPACK_ARCH256
+#undef UNPACK_ARCH256
 
 #if defined(ARROW_HAVE_AVX512) || defined(ARROW_HAVE_RUNTIME_AVX512)
 

cpp/src/arrow/util/bpacking_simd_internal.h

@@ -27,7 +27,7 @@
 #include "arrow/util/bpacking_scalar_internal.h"
 #include "arrow/util/bpacking_simd_internal.h"
 
-#if defined(ARROW_HAVE_RUNTIME_AVX2)
+#if defined(ARROW_HAVE_RUNTIME_AVX2) || defined(ARROW_HAVE_RUNTIME_SVE256)
 #  include "arrow/util/cpu_info.h"
 #endif
 
@@ -349,6 +349,39 @@ TEST_P(TestUnpack, Unpack32Neon) { this->TestAll(&bpacking::unpack_neon<uint32_t
 TEST_P(TestUnpack, Unpack64Neon) { this->TestAll(&bpacking::unpack_neon<uint64_t>); }
 #endif
 
+#if defined(ARROW_HAVE_RUNTIME_SVE256)
+TEST_P(TestUnpack, UnpackBoolSve256) {
+  if (!CpuInfo::GetInstance()->IsSupported(CpuInfo::SVE256)) {
+    GTEST_SKIP() << "Test requires SVE256";
+  }
+  this->TestAll(&bpacking::unpack_sve256<bool>);
+}
+TEST_P(TestUnpack, Unpack8Sve256) {
+  if (!CpuInfo::GetInstance()->IsSupported(CpuInfo::SVE256)) {
+    GTEST_SKIP() << "Test requires SVE256";
+  }
+  this->TestAll(&bpacking::unpack_sve256<uint8_t>);
+}
+TEST_P(TestUnpack, Unpack16Sve256) {
+  if (!CpuInfo::GetInstance()->IsSupported(CpuInfo::SVE256)) {
+    GTEST_SKIP() << "Test requires SVE256";
+  }
+  this->TestAll(&bpacking::unpack_sve256<uint16_t>);
+}
+TEST_P(TestUnpack, Unpack32Sve256) {
+  if (!CpuInfo::GetInstance()->IsSupported(CpuInfo::SVE256)) {
+    GTEST_SKIP() << "Test requires SVE256";
+  }
+  this->TestAll(&bpacking::unpack_sve256<uint32_t>);
+}
+TEST_P(TestUnpack, Unpack64Sve256) {
+  if (!CpuInfo::GetInstance()->IsSupported(CpuInfo::SVE256)) {
+    GTEST_SKIP() << "Test requires SVE256";
+  }
+  this->TestAll(&bpacking::unpack_sve256<uint64_t>);
+}
+#endif
+
 TEST_P(TestUnpack, UnpackBool) { this->TestAll(&unpack<bool>); }
 TEST_P(TestUnpack, Unpack8) { this->TestAll(&unpack<uint8_t>); }
 TEST_P(TestUnpack, Unpack16) { this->TestAll(&unpack<uint16_t>); }