From 87fccdeaa3c3cb2c1f910e5fca26261162c1c4b4 Mon Sep 17 00:00:00 2001 From: Fredrik Fornwall Date: Wed, 15 Jul 2026 11:19:26 +0200 Subject: [PATCH 1/2] Support float16 in hash kernels (dictionary_encode, unique, value_counts) dictionary_encode(), unique() and value_counts() failed on half_float input with: ArrowNotImplementedError: Function 'dictionary_encode' has no kernel matching input types (halffloat) AddHashKernels() registers kernels by iterating PrimitiveTypes(), which is derived from FloatingPointTypes() and only contains float32 and float64 -- float16 is absent, so no half_float kernel was ever registered. GetHashInit() also had no HALF_FLOAT case. Rather than widening the public FloatingPointTypes()/NumericTypes()/ PrimitiveTypes() lists (which many other kernel registrations consume, e.g. aggregate_basic.cc, and which would change behaviour well beyond this bug), register the float16 kernel explicitly in AddHashKernels() and dispatch HALF_FLOAT in GetHashInit(). HALF_FLOAT is hashed via RegularHashKernel, i.e. by its raw bit pattern. This mirrors the existing treatment of float32 (hashed via UInt32Type) and float64 (via UInt64Type), so NaN/-0.0 semantics are consistent with what those types already do. HalfFloatType::c_type is already uint16_t, and the hash kernel keeps the original type for its output, so the resulting dictionary is correctly typed float16. Signed-off-by: Fredrik Fornwall --- cpp/src/arrow/compute/kernels/vector_hash.cc | 8 ++++ .../arrow/compute/kernels/vector_hash_test.cc | 38 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/cpp/src/arrow/compute/kernels/vector_hash.cc b/cpp/src/arrow/compute/kernels/vector_hash.cc index 90ec9e365c35..f42972e18af0 100644 --- a/cpp/src/arrow/compute/kernels/vector_hash.cc +++ b/cpp/src/arrow/compute/kernels/vector_hash.cc @@ -550,6 +550,7 @@ KernelInit GetHashInit(Type::type type_id) { return HashInit>; case Type::INT16: case Type::UINT16: + case Type::HALF_FLOAT: return HashInit>; case Type::INT32: case Type::UINT32: @@ -700,6 +701,13 @@ void AddHashKernels(VectorFunction* func, VectorKernel base, OutputType out_ty) DCHECK_OK(func->AddKernel(base)); } + // float16() is not part of PrimitiveTypes() (FloatingPointTypes() only covers + // float32 and float64; see GH-43017), so it must be registered explicitly. Like + // float32 and float64, it is hashed by its raw bit pattern (via UInt16Type). + base.init = GetHashInit(Type::HALF_FLOAT); + base.signature = KernelSignature::Make({float16()}, out_ty); + DCHECK_OK(func->AddKernel(base)); + // Parametric types that we want matching to be dependent only on type id auto parametric_types = {Type::TIME32, Type::TIME64, Type::TIMESTAMP, Type::DURATION, Type::FIXED_SIZE_BINARY}; diff --git a/cpp/src/arrow/compute/kernels/vector_hash_test.cc b/cpp/src/arrow/compute/kernels/vector_hash_test.cc index b0fa296e0074..5b066ab782f4 100644 --- a/cpp/src/arrow/compute/kernels/vector_hash_test.cc +++ b/cpp/src/arrow/compute/kernels/vector_hash_test.cc @@ -403,6 +403,44 @@ TEST_F(TestHashKernel, DictEncodeBoolean) { ArrayFromJSON(boolean(), "[true]"), ArrayFromJSON(int32(), "[0, null, 0]")); } +// float16 is not part of PrimitiveTypes(), so it is not covered by +// TestHashKernelPrimitive above and gets its own coverage here. Like float32 and +// float64, it is hashed by its raw bit pattern. +TEST_F(TestHashKernel, UniqueHalfFloat) { + CheckUnique(ArrayFromJSON(float16(), "[1.5, 2.5, null, 1.5, 3.5, null]"), + ArrayFromJSON(float16(), "[1.5, 2.5, null, 3.5]")); + + // No nulls + CheckUnique(ArrayFromJSON(float16(), "[1.5, 2.5, 1.5, 3.5]"), + ArrayFromJSON(float16(), "[1.5, 2.5, 3.5]")); + + // Sliced + CheckUnique(ArrayFromJSON(float16(), "[1.5, 2.5, null, 3.5, 2.5, 1.5]")->Slice(1, 4), + ArrayFromJSON(float16(), "[2.5, null, 3.5]")); +} + +TEST_F(TestHashKernel, ValueCountsHalfFloat) { + CheckValueCounts(ArrayFromJSON(float16(), "[1.5, 2.5, null, 1.5, 3.5, null]"), + ArrayFromJSON(float16(), "[1.5, 2.5, null, 3.5]"), + ArrayFromJSON(int64(), "[2, 1, 2, 1]")); +} + +TEST_F(TestHashKernel, DictEncodeHalfFloat) { + CheckDictEncode(ArrayFromJSON(float16(), "[1.5, 2.5, null, 1.5, 3.5, null]"), + ArrayFromJSON(float16(), "[1.5, 2.5, 3.5]"), + ArrayFromJSON(int32(), "[0, 1, null, 0, 2, null]")); + + // No nulls + CheckDictEncode(ArrayFromJSON(float16(), "[1.5, 2.5, 1.5, 3.5]"), + ArrayFromJSON(float16(), "[1.5, 2.5, 3.5]"), + ArrayFromJSON(int32(), "[0, 1, 0, 2]")); + + // Sliced + CheckDictEncode( + ArrayFromJSON(float16(), "[1.5, 2.5, null, 3.5, 2.5, 1.5]")->Slice(1, 4), + ArrayFromJSON(float16(), "[2.5, 3.5]"), ArrayFromJSON(int32(), "[0, null, 1, 0]")); +} + template class TestHashKernelBinaryTypes : public TestHashKernel { protected: From 723f3f95f6b841acd4e207613150979fb0beefea Mon Sep 17 00:00:00 2001 From: Fredrik Fornwall Date: Sat, 12 Sep 2026 01:07:52 +0200 Subject: [PATCH 2/2] Cover HalfFloatType in TestHashKernelPrimitive TestHashKernelPrimitive takes a hand-written ::testing::Types list, not PrimitiveTypes(), so there was no reason for float16 to need its own tests (the earlier comment claiming otherwise was wrong). Adding HalfFloatType to the list drops 39 lines and gains coverage: the standalone tests only exercised Unique/ValueCounts/DictEncode, while the typed suite also runs ZeroChunks and PrimitiveResizeTable. Signed-off-by: Fredrik Fornwall --- .../arrow/compute/kernels/vector_hash_test.cc | 42 +------------------ 1 file changed, 2 insertions(+), 40 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/vector_hash_test.cc b/cpp/src/arrow/compute/kernels/vector_hash_test.cc index 5b066ab782f4..b11e0a722a57 100644 --- a/cpp/src/arrow/compute/kernels/vector_hash_test.cc +++ b/cpp/src/arrow/compute/kernels/vector_hash_test.cc @@ -150,8 +150,8 @@ template class TestHashKernelPrimitive : public ::testing::Test {}; typedef ::testing::Types + UInt32Type, Int64Type, UInt64Type, HalfFloatType, FloatType, + DoubleType, Date32Type, Date64Type> PrimitiveDictionaries; TYPED_TEST_SUITE(TestHashKernelPrimitive, PrimitiveDictionaries); @@ -403,44 +403,6 @@ TEST_F(TestHashKernel, DictEncodeBoolean) { ArrayFromJSON(boolean(), "[true]"), ArrayFromJSON(int32(), "[0, null, 0]")); } -// float16 is not part of PrimitiveTypes(), so it is not covered by -// TestHashKernelPrimitive above and gets its own coverage here. Like float32 and -// float64, it is hashed by its raw bit pattern. -TEST_F(TestHashKernel, UniqueHalfFloat) { - CheckUnique(ArrayFromJSON(float16(), "[1.5, 2.5, null, 1.5, 3.5, null]"), - ArrayFromJSON(float16(), "[1.5, 2.5, null, 3.5]")); - - // No nulls - CheckUnique(ArrayFromJSON(float16(), "[1.5, 2.5, 1.5, 3.5]"), - ArrayFromJSON(float16(), "[1.5, 2.5, 3.5]")); - - // Sliced - CheckUnique(ArrayFromJSON(float16(), "[1.5, 2.5, null, 3.5, 2.5, 1.5]")->Slice(1, 4), - ArrayFromJSON(float16(), "[2.5, null, 3.5]")); -} - -TEST_F(TestHashKernel, ValueCountsHalfFloat) { - CheckValueCounts(ArrayFromJSON(float16(), "[1.5, 2.5, null, 1.5, 3.5, null]"), - ArrayFromJSON(float16(), "[1.5, 2.5, null, 3.5]"), - ArrayFromJSON(int64(), "[2, 1, 2, 1]")); -} - -TEST_F(TestHashKernel, DictEncodeHalfFloat) { - CheckDictEncode(ArrayFromJSON(float16(), "[1.5, 2.5, null, 1.5, 3.5, null]"), - ArrayFromJSON(float16(), "[1.5, 2.5, 3.5]"), - ArrayFromJSON(int32(), "[0, 1, null, 0, 2, null]")); - - // No nulls - CheckDictEncode(ArrayFromJSON(float16(), "[1.5, 2.5, 1.5, 3.5]"), - ArrayFromJSON(float16(), "[1.5, 2.5, 3.5]"), - ArrayFromJSON(int32(), "[0, 1, 0, 2]")); - - // Sliced - CheckDictEncode( - ArrayFromJSON(float16(), "[1.5, 2.5, null, 3.5, 2.5, 1.5]")->Slice(1, 4), - ArrayFromJSON(float16(), "[2.5, 3.5]"), ArrayFromJSON(int32(), "[0, null, 1, 0]")); -} - template class TestHashKernelBinaryTypes : public TestHashKernel { protected: