[MOD-17844] Stop ARM SIMD tiers executing instructions the running CPU lacks

cmake/aarch64InstructionFlags.cmake

@@ -9,7 +9,8 @@ CHECK_CXX_COMPILER_FLAG("-march=armv8-a" CXX_ARMV8A)
 CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+dotprod" CXX_NEON_DOTPROD)
 CHECK_CXX_COMPILER_FLAG("-march=armv8-a+sve" CXX_SVE)
 CHECK_CXX_COMPILER_FLAG("-march=armv9-a+sve2" CXX_SVE2)
-CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+fp16fml" CXX_NEON_HP)
+CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+fp16" CXX_NEON_HP)
+CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+fp16fml" CXX_NEON_FHM)
 CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+bf16" CXX_NEON_BF16)
 CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+sve+bf16" CXX_SVE_BF16)
 
@@ -29,6 +30,10 @@ if (CXX_NEON_HP)
   message(STATUS "Using ARMv8.2-a with NEON half-percision extension")
   add_compile_definitions(OPT_NEON_HP)
 endif()
+if (CXX_NEON_FHM)
+  message(STATUS "Using ARMv8.2-a with NEON FHM extension")
+  add_compile_definitions(OPT_NEON_FHM)
+endif()
 if (CXX_NEON_BF16)
   add_compile_definitions(OPT_NEON_BF16)
 endif()

src/VecSim/spaces/CMakeLists.txt

@@ -141,10 +141,17 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)|(ARM64)|(armv.*)")
 	# NEON half-precision support
 	if (CXX_NEON_HP AND CXX_ARMV8A)
 		message("Building with NEON+HP")
-		set_source_files_properties(functions/NEON_HP.cpp PROPERTIES COMPILE_FLAGS "-march=armv8.2-a+fp16fml")
+		set_source_files_properties(functions/NEON_HP.cpp PROPERTIES COMPILE_FLAGS "-march=armv8.2-a+fp16")
 		list(APPEND OPTIMIZATIONS functions/NEON_HP.cpp)
   endif()
 
+	# NEON FHM (FEAT_FHM / asimdfhm) support
+	if (CXX_NEON_FHM AND CXX_ARMV8A)
+		message("Building with NEON+FHM")
+		set_source_files_properties(functions/NEON_FHM.cpp PROPERTIES COMPILE_FLAGS "-march=armv8.2-a+fp16fml")
+		list(APPEND OPTIMIZATIONS functions/NEON_FHM.cpp)
+	endif()
+
 	# NEON bfloat16 support
 	if (CXX_NEON_BF16)
 		message("Building with NEON + BF16")

src/VecSim/spaces/IP/IP_SVE_FP16.h

@@ -8,6 +8,12 @@
  */
 #include <arm_sve.h>
 
+// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The
+// anonymous namespace keeps each tier's bodies to itself; without it they are weak
+// symbols that both objects define and link order picks the -march. Only the Choose_*
+// entry points stay external. Dependencies above must stay outside the namespace.
+namespace {
+
 inline void InnerProduct_Step(const float16_t *vec1, const float16_t *vec2, svfloat16_t &acc,
                               size_t &offset, const size_t chunk) {
     svbool_t all = svptrue_b16();
@@ -72,3 +78,4 @@ float FP16_InnerProduct_SVE(const void *pVect1v, const void *pVect2v, size_t dim
     float result = svaddv_f16(all, acc1);
     return 1.0f - result;
 }
+} // namespace

src/VecSim/spaces/IP/IP_SVE_FP32.h

@@ -10,6 +10,12 @@
 
 #include <arm_sve.h>
 
+// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The
+// anonymous namespace keeps each tier's bodies to itself; without it they are weak
+// symbols that both objects define and link order picks the -march. Only the Choose_*
+// entry points stay external. Dependencies above must stay outside the namespace.
+namespace {
+
 static inline void InnerProductStep(float *&pVect1, float *&pVect2, size_t &offset,
                                     svfloat32_t &sum, const size_t chunk) {
     svfloat32_t v1 = svld1_f32(svptrue_b32(), pVect1 + offset);
@@ -77,3 +83,4 @@ float FP32_InnerProductSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t
     float result = svaddv_f32(svptrue_b32(), sum_all);
     return 1.0f - result;
 }
+} // namespace

src/VecSim/spaces/IP/IP_SVE_FP64.h

@@ -10,6 +10,12 @@
 
 #include <arm_sve.h>
 
+// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The
+// anonymous namespace keeps each tier's bodies to itself; without it they are weak
+// symbols that both objects define and link order picks the -march. Only the Choose_*
+// entry points stay external. Dependencies above must stay outside the namespace.
+namespace {
+
 inline void InnerProductStep(double *&pVect1, double *&pVect2, size_t &offset, svfloat64_t &sum,
                              const size_t chunk) {
     // Load vectors
@@ -73,3 +79,4 @@ double FP64_InnerProductSIMD_SVE(const void *pVect1v, const void *pVect2v, size_
     double result = svaddv_f64(svptrue_b64(), sum_all);
     return 1.0 - result;
 }
+} // namespace

src/VecSim/spaces/IP/IP_SVE_INT8.h

@@ -11,6 +11,12 @@
 #include "VecSim/spaces/space_includes.h"
 #include <arm_sve.h>
 
+// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The
+// anonymous namespace keeps each tier's bodies to itself; without it they are weak
+// symbols that both objects define and link order picks the -march. Only the Choose_*
+// entry points stay external. Dependencies above must stay outside the namespace.
+namespace {
+
 inline void InnerProductStep(const int8_t *&pVect1, const int8_t *&pVect2, size_t &offset,
                              svint32_t &sum, const size_t chunk) {
     svbool_t pg = svptrue_b8();
@@ -103,3 +109,4 @@ float INT8_CosineSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t dimen
     const float norm_v2 = load_unaligned<float>(static_cast<const int8_t *>(pVect2v) + dimension);
     return 1.0f - ip / (norm_v1 * norm_v2);
 }
+} // namespace

src/VecSim/spaces/IP/IP_SVE_SQ8_FP32.h

@@ -11,6 +11,12 @@
 #include "VecSim/types/sq8.h"
 #include <arm_sve.h>
 
+// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The
+// anonymous namespace keeps each tier's bodies to itself; without it they are weak
+// symbols that both objects define and link order picks the -march. Only the Choose_*
+// entry points stay external. Dependencies above must stay outside the namespace.
+namespace {
+
 using sq8 = vecsim_types::sq8;
 /*
  * Optimized asymmetric SQ8 inner product using algebraic identity:
@@ -143,3 +149,4 @@ float SQ8_FP32_CosineSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t d
     return SQ8_FP32_InnerProductSIMD_SVE<partial_chunk, additional_steps>(pVect1v, pVect2v,
                                                                           dimension);
 }
+} // namespace

src/VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h

@@ -12,6 +12,12 @@
 #include "VecSim/types/sq8.h"
 #include <arm_sve.h>
 
+// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The
+// anonymous namespace keeps each tier's bodies to itself; without it they are weak
+// symbols that both objects define and link order picks the -march. Only the Choose_*
+// entry points stay external. Dependencies above must stay outside the namespace.
+namespace {
+
 using sq8 = vecsim_types::sq8;
 
 /**
@@ -79,3 +85,4 @@ float SQ8_SQ8_CosineSIMD_SVE(const void *pVec1v, const void *pVec2v, size_t dime
     // Assume vectors are normalized.
     return SQ8_SQ8_InnerProductSIMD_SVE<partial_chunk, additional_steps>(pVec1v, pVec2v, dimension);
 }
+} // namespace

src/VecSim/spaces/IP/IP_SVE_UINT8.h

@@ -10,6 +10,12 @@
 #include "VecSim/spaces/space_includes.h"
 #include <arm_sve.h>
 
+// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The
+// anonymous namespace keeps each tier's bodies to itself; without it they are weak
+// symbols that both objects define and link order picks the -march. Only the Choose_*
+// entry points stay external. Dependencies above must stay outside the namespace.
+namespace {
+
 inline void InnerProductStep(const uint8_t *&pVect1, const uint8_t *&pVect2, size_t &offset,
                              svuint32_t &sum, const size_t chunk) {
     svbool_t pg = svptrue_b8();
@@ -102,3 +108,4 @@ float UINT8_CosineSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t dime
     const float norm_v2 = load_unaligned<float>(static_cast<const uint8_t *>(pVect2v) + dimension);
     return 1.0f - ip / (norm_v1 * norm_v2);
 }
+} // namespace

src/VecSim/spaces/IP_space.cpp

@@ -29,6 +29,7 @@
 #include "VecSim/spaces/functions/NEON.h"
 #include "VecSim/spaces/functions/NEON_DOTPROD.h"
 #include "VecSim/spaces/functions/NEON_HP.h"
+#include "VecSim/spaces/functions/NEON_FHM.h"
 #include "VecSim/spaces/functions/NEON_BF16.h"
 #include "VecSim/spaces/functions/SVE.h"
 #include "VecSim/spaces/functions/SVE_BF16.h"
@@ -241,10 +242,12 @@ dist_func_t<float> IP_SQ8_FP16_GetDistFunc(size_t dim, unsigned char *alignment,
         return Choose_SQ8_FP16_IP_implementation_SVE(dim);
     }
 #endif
-#ifdef OPT_NEON_HP
-    if (features.asimdfhm) {
+#ifdef OPT_NEON_FHM
+    if (features.asimdhp && features.asimdfhm) {
         return Choose_SQ8_FP16_IP_implementation_NEON_FHM(dim);
     }
+#endif
+#ifdef OPT_NEON_HP
     if (features.asimdhp) {
         return Choose_SQ8_FP16_IP_implementation_NEON_HP(dim);
     }
@@ -313,10 +316,12 @@ dist_func_t<float> Cosine_SQ8_FP16_GetDistFunc(size_t dim, unsigned char *alignm
         return Choose_SQ8_FP16_Cosine_implementation_SVE(dim);
     }
 #endif
-#ifdef OPT_NEON_HP
-    if (features.asimdfhm) {
+#ifdef OPT_NEON_FHM
+    if (features.asimdhp && features.asimdfhm) {
         return Choose_SQ8_FP16_Cosine_implementation_NEON_FHM(dim);
     }
+#endif
+#ifdef OPT_NEON_HP
     if (features.asimdhp) {
         return Choose_SQ8_FP16_Cosine_implementation_NEON_HP(dim);
     }

src/VecSim/spaces/L2/L2_SVE_FP16.h

@@ -8,6 +8,12 @@
  */
 #include <arm_sve.h>
 
+// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The
+// anonymous namespace keeps each tier's bodies to itself; without it they are weak
+// symbols that both objects define and link order picks the -march. Only the Choose_*
+// entry points stay external. Dependencies above must stay outside the namespace.
+namespace {
+
 inline void L2Sqr_Step(const float16_t *vec1, const float16_t *vec2, svfloat16_t &acc,
                        size_t &offset, const size_t chunk) {
     svbool_t all = svptrue_b16();
@@ -73,3 +79,4 @@ float FP16_L2Sqr_SVE(const void *pVect1v, const void *pVect2v, size_t dimension)
     float result = svaddv_f16(all, acc1);
     return result;
 }
+} // namespace

src/VecSim/spaces/L2/L2_SVE_FP32.h

@@ -9,6 +9,12 @@
 #include "VecSim/spaces/space_includes.h"
 #include <arm_sve.h>
 
+// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The
+// anonymous namespace keeps each tier's bodies to itself; without it they are weak
+// symbols that both objects define and link order picks the -march. Only the Choose_*
+// entry points stay external. Dependencies above must stay outside the namespace.
+namespace {
+
 static inline void L2SquareStep(float *&pVect1, float *&pVect2, size_t &offset, svfloat32_t &sum,
                                 const size_t chunk) {
     // Load vectors
@@ -87,3 +93,4 @@ float FP32_L2SqrSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t dimens
     float result = svaddv_f32(svptrue_b32(), sum_all);
     return result;
 }
+} // namespace

src/VecSim/spaces/L2/L2_SVE_FP64.h

@@ -9,6 +9,12 @@
 #include "VecSim/spaces/space_includes.h"
 #include <arm_sve.h>
 
+// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The
+// anonymous namespace keeps each tier's bodies to itself; without it they are weak
+// symbols that both objects define and link order picks the -march. Only the Choose_*
+// entry points stay external. Dependencies above must stay outside the namespace.
+namespace {
+
 inline void L2SquareStep(double *&pVect1, double *&pVect2, size_t &offset, svfloat64_t &sum,
                          const size_t chunk) {
     // Load vectors
@@ -81,3 +87,4 @@ double FP64_L2SqrSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t dimen
     double result = svaddv_f64(svptrue_b64(), sum_all);
     return result;
 }
+} // namespace

src/VecSim/spaces/L2/L2_SVE_INT8.h

@@ -9,6 +9,12 @@
 #include "VecSim/spaces/space_includes.h"
 #include <arm_sve.h>
 
+// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The
+// anonymous namespace keeps each tier's bodies to itself; without it they are weak
+// symbols that both objects define and link order picks the -march. Only the Choose_*
+// entry points stay external. Dependencies above must stay outside the namespace.
+namespace {
+
 // Aligned step using svptrue_b8()
 inline void L2SquareStep(const int8_t *&pVect1, const int8_t *&pVect2, size_t &offset,
                          svuint32_t &sum, const size_t chunk) {
@@ -89,3 +95,4 @@ float INT8_L2SqrSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t dimens
     svuint32_t sum_all = svadd_u32_x(all, sum0, sum2);
     return svaddv_u32(svptrue_b32(), sum_all);
 }
+} // namespace

src/VecSim/spaces/L2/L2_SVE_SQ8_FP32.h

@@ -12,6 +12,12 @@
 #include "VecSim/types/sq8.h"
 #include <arm_sve.h>
 
+// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The
+// anonymous namespace keeps each tier's bodies to itself; without it they are weak
+// symbols that both objects define and link order picks the -march. Only the Choose_*
+// entry points stay external. Dependencies above must stay outside the namespace.
+namespace {
+
 using sq8 = vecsim_types::sq8;
 
 /*
@@ -46,3 +52,4 @@ float SQ8_FP32_L2SqrSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t di
     // L2² = ||x||² + ||y||² - 2*IP(x, y)
     return x_sum_sq + y_sum_sq - 2.0f * ip;
 }
+} // namespace

src/VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h

@@ -11,6 +11,12 @@
 #include "VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h"
 #include "VecSim/types/sq8.h"
 
+// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The
+// anonymous namespace keeps each tier's bodies to itself; without it they are weak
+// symbols that both objects define and link order picks the -march. Only the Choose_*
+// entry points stay external. Dependencies above must stay outside the namespace.
+namespace {
+
 using sq8 = vecsim_types::sq8;
 
 /**
@@ -45,3 +51,4 @@ float SQ8_SQ8_L2SqrSIMD_SVE(const void *pVec1v, const void *pVec2v, size_t dimen
     // L2² = ||x||² + ||y||² - 2*IP(x, y)
     return sum_sq_1 + sum_sq_2 - 2.0f * ip;
 }
+} // namespace

src/VecSim/spaces/L2/L2_SVE_UINT8.h

@@ -9,6 +9,12 @@
 #include "VecSim/spaces/space_includes.h"
 #include <arm_sve.h>
 
+// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The
+// anonymous namespace keeps each tier's bodies to itself; without it they are weak
+// symbols that both objects define and link order picks the -march. Only the Choose_*
+// entry points stay external. Dependencies above must stay outside the namespace.
+namespace {
+
 // Aligned step using svptrue_b8()
 inline void L2SquareStep(const uint8_t *&pVect1, const uint8_t *&pVect2, size_t &offset,
                          svuint32_t &sum, const size_t chunk) {
@@ -87,3 +93,4 @@ float UINT8_L2SqrSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t dimen
     svuint32_t sum_all = svadd_u32_x(all, sum0, sum2);
     return svaddv_u32(svptrue_b32(), sum_all);
 }
+} // namespace

src/VecSim/spaces/L2_space.cpp

@@ -28,6 +28,7 @@
 #include "VecSim/spaces/functions/NEON.h"
 #include "VecSim/spaces/functions/NEON_DOTPROD.h"
 #include "VecSim/spaces/functions/NEON_HP.h"
+#include "VecSim/spaces/functions/NEON_FHM.h"
 #include "VecSim/spaces/functions/NEON_BF16.h"
 #include "VecSim/spaces/functions/SVE.h"
 #include "VecSim/spaces/functions/SVE_BF16.h"
@@ -170,10 +171,12 @@ dist_func_t<float> L2_SQ8_FP16_GetDistFunc(size_t dim, unsigned char *alignment,
         return Choose_SQ8_FP16_L2_implementation_SVE(dim);
     }
 #endif
-#ifdef OPT_NEON_HP
-    if (features.asimdfhm) {
+#ifdef OPT_NEON_FHM
+    if (features.asimdhp && features.asimdfhm) {
         return Choose_SQ8_FP16_L2_implementation_NEON_FHM(dim);
     }
+#endif
+#ifdef OPT_NEON_HP
     if (features.asimdhp) {
         return Choose_SQ8_FP16_L2_implementation_NEON_HP(dim);
     }

src/VecSim/spaces/functions/NEON_FHM.cpp

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2006-Present, Redis Ltd.
+ * All rights reserved.
+ *
+ * Licensed under your choice of the Redis Source Available License 2.0
+ * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
+ * GNU Affero General Public License v3 (AGPLv3).
+ */
+#include "NEON_FHM.h"
+
+#include "VecSim/spaces/IP/IP_NEON_SQ8_FP16.h"
+#include "VecSim/spaces/L2/L2_NEON_SQ8_FP16.h"
+
+namespace spaces {
+
+#include "implementation_chooser.h"
+
+dist_func_t<float> Choose_SQ8_FP16_IP_implementation_NEON_FHM(size_t dim) {
+    dist_func_t<float> ret_dist_func;
+    CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_InnerProductSIMD16_NEON_FHM);
+    return ret_dist_func;
+}
+
+dist_func_t<float> Choose_SQ8_FP16_L2_implementation_NEON_FHM(size_t dim) {
+    dist_func_t<float> ret_dist_func;
+    CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_L2SqrSIMD16_NEON_FHM);
+    return ret_dist_func;
+}
+
+dist_func_t<float> Choose_SQ8_FP16_Cosine_implementation_NEON_FHM(size_t dim) {
+    dist_func_t<float> ret_dist_func;
+    CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_CosineSIMD16_NEON_FHM);
+    return ret_dist_func;
+}
+
+#include "implementation_chooser_cleanup.h"
+
+} // namespace spaces