diff --git a/cmake/aarch64InstructionFlags.cmake b/cmake/aarch64InstructionFlags.cmake index 1a2842265..5cd84331a 100644 --- a/cmake/aarch64InstructionFlags.cmake +++ b/cmake/aarch64InstructionFlags.cmake @@ -9,7 +9,8 @@ CHECK_CXX_COMPILER_FLAG("-march=armv8-a" CXX_ARMV8A) CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+dotprod" CXX_NEON_DOTPROD) CHECK_CXX_COMPILER_FLAG("-march=armv8-a+sve" CXX_SVE) CHECK_CXX_COMPILER_FLAG("-march=armv9-a+sve2" CXX_SVE2) -CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+fp16fml" CXX_NEON_HP) +CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+fp16" CXX_NEON_HP) +CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+fp16fml" CXX_NEON_FHM) CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+bf16" CXX_NEON_BF16) CHECK_CXX_COMPILER_FLAG("-march=armv8.2-a+sve+bf16" CXX_SVE_BF16) @@ -29,6 +30,10 @@ if (CXX_NEON_HP) message(STATUS "Using ARMv8.2-a with NEON half-percision extension") add_compile_definitions(OPT_NEON_HP) endif() +if (CXX_NEON_FHM) + message(STATUS "Using ARMv8.2-a with NEON FHM extension") + add_compile_definitions(OPT_NEON_FHM) +endif() if (CXX_NEON_BF16) add_compile_definitions(OPT_NEON_BF16) endif() diff --git a/src/VecSim/spaces/CMakeLists.txt b/src/VecSim/spaces/CMakeLists.txt index 309d3f3a4..eb8a15fdd 100644 --- a/src/VecSim/spaces/CMakeLists.txt +++ b/src/VecSim/spaces/CMakeLists.txt @@ -141,10 +141,17 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)|(ARM64)|(armv.*)") # NEON half-precision support if (CXX_NEON_HP AND CXX_ARMV8A) message("Building with NEON+HP") - set_source_files_properties(functions/NEON_HP.cpp PROPERTIES COMPILE_FLAGS "-march=armv8.2-a+fp16fml") + set_source_files_properties(functions/NEON_HP.cpp PROPERTIES COMPILE_FLAGS "-march=armv8.2-a+fp16") list(APPEND OPTIMIZATIONS functions/NEON_HP.cpp) endif() + # NEON FHM (FEAT_FHM / asimdfhm) support + if (CXX_NEON_FHM AND CXX_ARMV8A) + message("Building with NEON+FHM") + set_source_files_properties(functions/NEON_FHM.cpp PROPERTIES COMPILE_FLAGS "-march=armv8.2-a+fp16fml") + list(APPEND OPTIMIZATIONS functions/NEON_FHM.cpp) + endif() + # NEON bfloat16 support if (CXX_NEON_BF16) message("Building with NEON + BF16") diff --git a/src/VecSim/spaces/IP/IP_SVE_FP16.h b/src/VecSim/spaces/IP/IP_SVE_FP16.h index ac464977e..aaaf9dc0e 100644 --- a/src/VecSim/spaces/IP/IP_SVE_FP16.h +++ b/src/VecSim/spaces/IP/IP_SVE_FP16.h @@ -8,6 +8,12 @@ */ #include +// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The +// anonymous namespace keeps each tier's bodies to itself; without it they are weak +// symbols that both objects define and link order picks the -march. Only the Choose_* +// entry points stay external. Dependencies above must stay outside the namespace. +namespace { + inline void InnerProduct_Step(const float16_t *vec1, const float16_t *vec2, svfloat16_t &acc, size_t &offset, const size_t chunk) { svbool_t all = svptrue_b16(); @@ -72,3 +78,4 @@ float FP16_InnerProduct_SVE(const void *pVect1v, const void *pVect2v, size_t dim float result = svaddv_f16(all, acc1); return 1.0f - result; } +} // namespace diff --git a/src/VecSim/spaces/IP/IP_SVE_FP32.h b/src/VecSim/spaces/IP/IP_SVE_FP32.h index c1cc79ccd..f7808b017 100644 --- a/src/VecSim/spaces/IP/IP_SVE_FP32.h +++ b/src/VecSim/spaces/IP/IP_SVE_FP32.h @@ -10,6 +10,12 @@ #include +// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The +// anonymous namespace keeps each tier's bodies to itself; without it they are weak +// symbols that both objects define and link order picks the -march. Only the Choose_* +// entry points stay external. Dependencies above must stay outside the namespace. +namespace { + static inline void InnerProductStep(float *&pVect1, float *&pVect2, size_t &offset, svfloat32_t &sum, const size_t chunk) { svfloat32_t v1 = svld1_f32(svptrue_b32(), pVect1 + offset); @@ -77,3 +83,4 @@ float FP32_InnerProductSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t float result = svaddv_f32(svptrue_b32(), sum_all); return 1.0f - result; } +} // namespace diff --git a/src/VecSim/spaces/IP/IP_SVE_FP64.h b/src/VecSim/spaces/IP/IP_SVE_FP64.h index 1e091e85c..3c43325d1 100644 --- a/src/VecSim/spaces/IP/IP_SVE_FP64.h +++ b/src/VecSim/spaces/IP/IP_SVE_FP64.h @@ -10,6 +10,12 @@ #include +// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The +// anonymous namespace keeps each tier's bodies to itself; without it they are weak +// symbols that both objects define and link order picks the -march. Only the Choose_* +// entry points stay external. Dependencies above must stay outside the namespace. +namespace { + inline void InnerProductStep(double *&pVect1, double *&pVect2, size_t &offset, svfloat64_t &sum, const size_t chunk) { // Load vectors @@ -73,3 +79,4 @@ double FP64_InnerProductSIMD_SVE(const void *pVect1v, const void *pVect2v, size_ double result = svaddv_f64(svptrue_b64(), sum_all); return 1.0 - result; } +} // namespace diff --git a/src/VecSim/spaces/IP/IP_SVE_INT8.h b/src/VecSim/spaces/IP/IP_SVE_INT8.h index 62f55381b..817749543 100644 --- a/src/VecSim/spaces/IP/IP_SVE_INT8.h +++ b/src/VecSim/spaces/IP/IP_SVE_INT8.h @@ -11,6 +11,12 @@ #include "VecSim/spaces/space_includes.h" #include +// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The +// anonymous namespace keeps each tier's bodies to itself; without it they are weak +// symbols that both objects define and link order picks the -march. Only the Choose_* +// entry points stay external. Dependencies above must stay outside the namespace. +namespace { + inline void InnerProductStep(const int8_t *&pVect1, const int8_t *&pVect2, size_t &offset, svint32_t &sum, const size_t chunk) { svbool_t pg = svptrue_b8(); @@ -103,3 +109,4 @@ float INT8_CosineSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t dimen const float norm_v2 = load_unaligned(static_cast(pVect2v) + dimension); return 1.0f - ip / (norm_v1 * norm_v2); } +} // namespace diff --git a/src/VecSim/spaces/IP/IP_SVE_SQ8_FP32.h b/src/VecSim/spaces/IP/IP_SVE_SQ8_FP32.h index 07e47a07e..841ebb792 100644 --- a/src/VecSim/spaces/IP/IP_SVE_SQ8_FP32.h +++ b/src/VecSim/spaces/IP/IP_SVE_SQ8_FP32.h @@ -11,6 +11,12 @@ #include "VecSim/types/sq8.h" #include +// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The +// anonymous namespace keeps each tier's bodies to itself; without it they are weak +// symbols that both objects define and link order picks the -march. Only the Choose_* +// entry points stay external. Dependencies above must stay outside the namespace. +namespace { + using sq8 = vecsim_types::sq8; /* * Optimized asymmetric SQ8 inner product using algebraic identity: @@ -143,3 +149,4 @@ float SQ8_FP32_CosineSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t d return SQ8_FP32_InnerProductSIMD_SVE(pVect1v, pVect2v, dimension); } +} // namespace diff --git a/src/VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h b/src/VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h index 93ddb76cb..46f7911db 100644 --- a/src/VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h +++ b/src/VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h @@ -12,6 +12,12 @@ #include "VecSim/types/sq8.h" #include +// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The +// anonymous namespace keeps each tier's bodies to itself; without it they are weak +// symbols that both objects define and link order picks the -march. Only the Choose_* +// entry points stay external. Dependencies above must stay outside the namespace. +namespace { + using sq8 = vecsim_types::sq8; /** @@ -79,3 +85,4 @@ float SQ8_SQ8_CosineSIMD_SVE(const void *pVec1v, const void *pVec2v, size_t dime // Assume vectors are normalized. return SQ8_SQ8_InnerProductSIMD_SVE(pVec1v, pVec2v, dimension); } +} // namespace diff --git a/src/VecSim/spaces/IP/IP_SVE_UINT8.h b/src/VecSim/spaces/IP/IP_SVE_UINT8.h index 435a82d45..e64a9a698 100644 --- a/src/VecSim/spaces/IP/IP_SVE_UINT8.h +++ b/src/VecSim/spaces/IP/IP_SVE_UINT8.h @@ -10,6 +10,12 @@ #include "VecSim/spaces/space_includes.h" #include +// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The +// anonymous namespace keeps each tier's bodies to itself; without it they are weak +// symbols that both objects define and link order picks the -march. Only the Choose_* +// entry points stay external. Dependencies above must stay outside the namespace. +namespace { + inline void InnerProductStep(const uint8_t *&pVect1, const uint8_t *&pVect2, size_t &offset, svuint32_t &sum, const size_t chunk) { svbool_t pg = svptrue_b8(); @@ -102,3 +108,4 @@ float UINT8_CosineSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t dime const float norm_v2 = load_unaligned(static_cast(pVect2v) + dimension); return 1.0f - ip / (norm_v1 * norm_v2); } +} // namespace diff --git a/src/VecSim/spaces/IP_space.cpp b/src/VecSim/spaces/IP_space.cpp index 1f5ee55c2..b7f963d2c 100644 --- a/src/VecSim/spaces/IP_space.cpp +++ b/src/VecSim/spaces/IP_space.cpp @@ -29,6 +29,7 @@ #include "VecSim/spaces/functions/NEON.h" #include "VecSim/spaces/functions/NEON_DOTPROD.h" #include "VecSim/spaces/functions/NEON_HP.h" +#include "VecSim/spaces/functions/NEON_FHM.h" #include "VecSim/spaces/functions/NEON_BF16.h" #include "VecSim/spaces/functions/SVE.h" #include "VecSim/spaces/functions/SVE_BF16.h" @@ -241,10 +242,12 @@ dist_func_t IP_SQ8_FP16_GetDistFunc(size_t dim, unsigned char *alignment, return Choose_SQ8_FP16_IP_implementation_SVE(dim); } #endif -#ifdef OPT_NEON_HP - if (features.asimdfhm) { +#ifdef OPT_NEON_FHM + if (features.asimdhp && features.asimdfhm) { return Choose_SQ8_FP16_IP_implementation_NEON_FHM(dim); } +#endif +#ifdef OPT_NEON_HP if (features.asimdhp) { return Choose_SQ8_FP16_IP_implementation_NEON_HP(dim); } @@ -313,10 +316,12 @@ dist_func_t Cosine_SQ8_FP16_GetDistFunc(size_t dim, unsigned char *alignm return Choose_SQ8_FP16_Cosine_implementation_SVE(dim); } #endif -#ifdef OPT_NEON_HP - if (features.asimdfhm) { +#ifdef OPT_NEON_FHM + if (features.asimdhp && features.asimdfhm) { return Choose_SQ8_FP16_Cosine_implementation_NEON_FHM(dim); } +#endif +#ifdef OPT_NEON_HP if (features.asimdhp) { return Choose_SQ8_FP16_Cosine_implementation_NEON_HP(dim); } diff --git a/src/VecSim/spaces/L2/L2_SVE_FP16.h b/src/VecSim/spaces/L2/L2_SVE_FP16.h index 24b5ee2df..b68b8120a 100644 --- a/src/VecSim/spaces/L2/L2_SVE_FP16.h +++ b/src/VecSim/spaces/L2/L2_SVE_FP16.h @@ -8,6 +8,12 @@ */ #include +// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The +// anonymous namespace keeps each tier's bodies to itself; without it they are weak +// symbols that both objects define and link order picks the -march. Only the Choose_* +// entry points stay external. Dependencies above must stay outside the namespace. +namespace { + inline void L2Sqr_Step(const float16_t *vec1, const float16_t *vec2, svfloat16_t &acc, size_t &offset, const size_t chunk) { svbool_t all = svptrue_b16(); @@ -73,3 +79,4 @@ float FP16_L2Sqr_SVE(const void *pVect1v, const void *pVect2v, size_t dimension) float result = svaddv_f16(all, acc1); return result; } +} // namespace diff --git a/src/VecSim/spaces/L2/L2_SVE_FP32.h b/src/VecSim/spaces/L2/L2_SVE_FP32.h index 8367baa97..36445bf1e 100644 --- a/src/VecSim/spaces/L2/L2_SVE_FP32.h +++ b/src/VecSim/spaces/L2/L2_SVE_FP32.h @@ -9,6 +9,12 @@ #include "VecSim/spaces/space_includes.h" #include +// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The +// anonymous namespace keeps each tier's bodies to itself; without it they are weak +// symbols that both objects define and link order picks the -march. Only the Choose_* +// entry points stay external. Dependencies above must stay outside the namespace. +namespace { + static inline void L2SquareStep(float *&pVect1, float *&pVect2, size_t &offset, svfloat32_t &sum, const size_t chunk) { // Load vectors @@ -87,3 +93,4 @@ float FP32_L2SqrSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t dimens float result = svaddv_f32(svptrue_b32(), sum_all); return result; } +} // namespace diff --git a/src/VecSim/spaces/L2/L2_SVE_FP64.h b/src/VecSim/spaces/L2/L2_SVE_FP64.h index 8fb822544..0990751a6 100644 --- a/src/VecSim/spaces/L2/L2_SVE_FP64.h +++ b/src/VecSim/spaces/L2/L2_SVE_FP64.h @@ -9,6 +9,12 @@ #include "VecSim/spaces/space_includes.h" #include +// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The +// anonymous namespace keeps each tier's bodies to itself; without it they are weak +// symbols that both objects define and link order picks the -march. Only the Choose_* +// entry points stay external. Dependencies above must stay outside the namespace. +namespace { + inline void L2SquareStep(double *&pVect1, double *&pVect2, size_t &offset, svfloat64_t &sum, const size_t chunk) { // Load vectors @@ -81,3 +87,4 @@ double FP64_L2SqrSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t dimen double result = svaddv_f64(svptrue_b64(), sum_all); return result; } +} // namespace diff --git a/src/VecSim/spaces/L2/L2_SVE_INT8.h b/src/VecSim/spaces/L2/L2_SVE_INT8.h index af959d19a..ee829ba62 100644 --- a/src/VecSim/spaces/L2/L2_SVE_INT8.h +++ b/src/VecSim/spaces/L2/L2_SVE_INT8.h @@ -9,6 +9,12 @@ #include "VecSim/spaces/space_includes.h" #include +// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The +// anonymous namespace keeps each tier's bodies to itself; without it they are weak +// symbols that both objects define and link order picks the -march. Only the Choose_* +// entry points stay external. Dependencies above must stay outside the namespace. +namespace { + // Aligned step using svptrue_b8() inline void L2SquareStep(const int8_t *&pVect1, const int8_t *&pVect2, size_t &offset, svuint32_t &sum, const size_t chunk) { @@ -89,3 +95,4 @@ float INT8_L2SqrSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t dimens svuint32_t sum_all = svadd_u32_x(all, sum0, sum2); return svaddv_u32(svptrue_b32(), sum_all); } +} // namespace diff --git a/src/VecSim/spaces/L2/L2_SVE_SQ8_FP32.h b/src/VecSim/spaces/L2/L2_SVE_SQ8_FP32.h index 3f95c4ae4..a1a800727 100644 --- a/src/VecSim/spaces/L2/L2_SVE_SQ8_FP32.h +++ b/src/VecSim/spaces/L2/L2_SVE_SQ8_FP32.h @@ -12,6 +12,12 @@ #include "VecSim/types/sq8.h" #include +// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The +// anonymous namespace keeps each tier's bodies to itself; without it they are weak +// symbols that both objects define and link order picks the -march. Only the Choose_* +// entry points stay external. Dependencies above must stay outside the namespace. +namespace { + using sq8 = vecsim_types::sq8; /* @@ -46,3 +52,4 @@ float SQ8_FP32_L2SqrSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t di // L2² = ||x||² + ||y||² - 2*IP(x, y) return x_sum_sq + y_sum_sq - 2.0f * ip; } +} // namespace diff --git a/src/VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h b/src/VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h index 5fe194d80..7451904cb 100644 --- a/src/VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h +++ b/src/VecSim/spaces/L2/L2_SVE_SQ8_SQ8.h @@ -11,6 +11,12 @@ #include "VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h" #include "VecSim/types/sq8.h" +// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The +// anonymous namespace keeps each tier's bodies to itself; without it they are weak +// symbols that both objects define and link order picks the -march. Only the Choose_* +// entry points stay external. Dependencies above must stay outside the namespace. +namespace { + using sq8 = vecsim_types::sq8; /** @@ -45,3 +51,4 @@ float SQ8_SQ8_L2SqrSIMD_SVE(const void *pVec1v, const void *pVec2v, size_t dimen // L2² = ||x||² + ||y||² - 2*IP(x, y) return sum_sq_1 + sum_sq_2 - 2.0f * ip; } +} // namespace diff --git a/src/VecSim/spaces/L2/L2_SVE_UINT8.h b/src/VecSim/spaces/L2/L2_SVE_UINT8.h index 553db2169..bb52482ad 100644 --- a/src/VecSim/spaces/L2/L2_SVE_UINT8.h +++ b/src/VecSim/spaces/L2/L2_SVE_UINT8.h @@ -9,6 +9,12 @@ #include "VecSim/spaces/space_includes.h" #include +// SVE.cpp and SVE2.cpp both compile this header, under different -march flags. The +// anonymous namespace keeps each tier's bodies to itself; without it they are weak +// symbols that both objects define and link order picks the -march. Only the Choose_* +// entry points stay external. Dependencies above must stay outside the namespace. +namespace { + // Aligned step using svptrue_b8() inline void L2SquareStep(const uint8_t *&pVect1, const uint8_t *&pVect2, size_t &offset, svuint32_t &sum, const size_t chunk) { @@ -87,3 +93,4 @@ float UINT8_L2SqrSIMD_SVE(const void *pVect1v, const void *pVect2v, size_t dimen svuint32_t sum_all = svadd_u32_x(all, sum0, sum2); return svaddv_u32(svptrue_b32(), sum_all); } +} // namespace diff --git a/src/VecSim/spaces/L2_space.cpp b/src/VecSim/spaces/L2_space.cpp index cf0b52f7c..2c9223b67 100644 --- a/src/VecSim/spaces/L2_space.cpp +++ b/src/VecSim/spaces/L2_space.cpp @@ -28,6 +28,7 @@ #include "VecSim/spaces/functions/NEON.h" #include "VecSim/spaces/functions/NEON_DOTPROD.h" #include "VecSim/spaces/functions/NEON_HP.h" +#include "VecSim/spaces/functions/NEON_FHM.h" #include "VecSim/spaces/functions/NEON_BF16.h" #include "VecSim/spaces/functions/SVE.h" #include "VecSim/spaces/functions/SVE_BF16.h" @@ -170,10 +171,12 @@ dist_func_t L2_SQ8_FP16_GetDistFunc(size_t dim, unsigned char *alignment, return Choose_SQ8_FP16_L2_implementation_SVE(dim); } #endif -#ifdef OPT_NEON_HP - if (features.asimdfhm) { +#ifdef OPT_NEON_FHM + if (features.asimdhp && features.asimdfhm) { return Choose_SQ8_FP16_L2_implementation_NEON_FHM(dim); } +#endif +#ifdef OPT_NEON_HP if (features.asimdhp) { return Choose_SQ8_FP16_L2_implementation_NEON_HP(dim); } diff --git a/src/VecSim/spaces/functions/NEON_FHM.cpp b/src/VecSim/spaces/functions/NEON_FHM.cpp new file mode 100644 index 000000000..2beceed0c --- /dev/null +++ b/src/VecSim/spaces/functions/NEON_FHM.cpp @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#include "NEON_FHM.h" + +#include "VecSim/spaces/IP/IP_NEON_SQ8_FP16.h" +#include "VecSim/spaces/L2/L2_NEON_SQ8_FP16.h" + +namespace spaces { + +#include "implementation_chooser.h" + +dist_func_t Choose_SQ8_FP16_IP_implementation_NEON_FHM(size_t dim) { + dist_func_t ret_dist_func; + CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_InnerProductSIMD16_NEON_FHM); + return ret_dist_func; +} + +dist_func_t Choose_SQ8_FP16_L2_implementation_NEON_FHM(size_t dim) { + dist_func_t ret_dist_func; + CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_L2SqrSIMD16_NEON_FHM); + return ret_dist_func; +} + +dist_func_t Choose_SQ8_FP16_Cosine_implementation_NEON_FHM(size_t dim) { + dist_func_t ret_dist_func; + CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_CosineSIMD16_NEON_FHM); + return ret_dist_func; +} + +#include "implementation_chooser_cleanup.h" + +} // namespace spaces diff --git a/src/VecSim/spaces/functions/NEON_FHM.h b/src/VecSim/spaces/functions/NEON_FHM.h new file mode 100644 index 000000000..0993aa69f --- /dev/null +++ b/src/VecSim/spaces/functions/NEON_FHM.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include "VecSim/spaces/spaces.h" + +namespace spaces { + +dist_func_t Choose_SQ8_FP16_IP_implementation_NEON_FHM(size_t dim); +dist_func_t Choose_SQ8_FP16_L2_implementation_NEON_FHM(size_t dim); +dist_func_t Choose_SQ8_FP16_Cosine_implementation_NEON_FHM(size_t dim); + +} // namespace spaces diff --git a/src/VecSim/spaces/functions/NEON_HP.cpp b/src/VecSim/spaces/functions/NEON_HP.cpp index 15e40ba82..20d93a517 100644 --- a/src/VecSim/spaces/functions/NEON_HP.cpp +++ b/src/VecSim/spaces/functions/NEON_HP.cpp @@ -47,25 +47,6 @@ dist_func_t Choose_SQ8_FP16_Cosine_implementation_NEON_HP(size_t dim) { return ret_dist_func; } -// FMLAL (FEAT_FHM / asimdfhm) variants. -dist_func_t Choose_SQ8_FP16_IP_implementation_NEON_FHM(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_InnerProductSIMD16_NEON_FHM); - return ret_dist_func; -} - -dist_func_t Choose_SQ8_FP16_L2_implementation_NEON_FHM(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_L2SqrSIMD16_NEON_FHM); - return ret_dist_func; -} - -dist_func_t Choose_SQ8_FP16_Cosine_implementation_NEON_FHM(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_CosineSIMD16_NEON_FHM); - return ret_dist_func; -} - #include "implementation_chooser_cleanup.h" } // namespace spaces diff --git a/src/VecSim/spaces/functions/NEON_HP.h b/src/VecSim/spaces/functions/NEON_HP.h index 83579d2b7..889eb0919 100644 --- a/src/VecSim/spaces/functions/NEON_HP.h +++ b/src/VecSim/spaces/functions/NEON_HP.h @@ -20,8 +20,4 @@ dist_func_t Choose_SQ8_FP16_IP_implementation_NEON_HP(size_t dim); dist_func_t Choose_SQ8_FP16_L2_implementation_NEON_HP(size_t dim); dist_func_t Choose_SQ8_FP16_Cosine_implementation_NEON_HP(size_t dim); -dist_func_t Choose_SQ8_FP16_IP_implementation_NEON_FHM(size_t dim); -dist_func_t Choose_SQ8_FP16_L2_implementation_NEON_FHM(size_t dim); -dist_func_t Choose_SQ8_FP16_Cosine_implementation_NEON_FHM(size_t dim); - } // namespace spaces diff --git a/tests/benchmark/spaces_benchmarks/bm_spaces.h b/tests/benchmark/spaces_benchmarks/bm_spaces.h index 2303eac0a..38995e35f 100644 --- a/tests/benchmark/spaces_benchmarks/bm_spaces.h +++ b/tests/benchmark/spaces_benchmarks/bm_spaces.h @@ -35,6 +35,7 @@ #include "VecSim/spaces/functions/NEON.h" #include "VecSim/spaces/functions/NEON_DOTPROD.h" #include "VecSim/spaces/functions/NEON_HP.h" +#include "VecSim/spaces/functions/NEON_FHM.h" #include "VecSim/spaces/functions/NEON_BF16.h" #include "VecSim/spaces/functions/SVE.h" #include "VecSim/spaces/functions/SVE_BF16.h" diff --git a/tests/benchmark/spaces_benchmarks/bm_spaces_sq8_fp16.cpp b/tests/benchmark/spaces_benchmarks/bm_spaces_sq8_fp16.cpp index 5ab529372..c7450264b 100644 --- a/tests/benchmark/spaces_benchmarks/bm_spaces_sq8_fp16.cpp +++ b/tests/benchmark/spaces_benchmarks/bm_spaces_sq8_fp16.cpp @@ -105,8 +105,10 @@ bool neon_hp_supported = arm_opt.asimdhp; INITIALIZE_BENCHMARKS_SET_L2_IP(BM_VecSimSpaces_SQ8_FP16, SQ8_FP16, NEON_HP, 16, neon_hp_supported); INITIALIZE_BENCHMARKS_SET_Cosine(BM_VecSimSpaces_SQ8_FP16, SQ8_FP16, NEON_HP, 16, neon_hp_supported); +#endif -bool neon_fhm_supported = arm_opt.asimdfhm; +#ifdef OPT_NEON_FHM +bool neon_fhm_supported = arm_opt.asimdhp && arm_opt.asimdfhm; INITIALIZE_BENCHMARKS_SET_L2_IP(BM_VecSimSpaces_SQ8_FP16, SQ8_FP16, NEON_FHM, 16, neon_fhm_supported); INITIALIZE_BENCHMARKS_SET_Cosine(BM_VecSimSpaces_SQ8_FP16, SQ8_FP16, NEON_FHM, 16, diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index c3e1cc987..4c72bcba6 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -81,3 +81,14 @@ gtest_discover_tests(test_uint8 TEST_PREFIX UINT8UNIT_) gtest_discover_tests(test_index_test_utils) # SVS Tiered tests are slow in debug mode, so we set a longer timeout gtest_discover_tests(test_svs PROPERTIES TIMEOUT 3000) + +# Guard the per-tier SIMD objects against sharing externally visible symbols, which would let +# link order decide which -march a kernel body was built for. See check_tier_linkage.py. +find_program(PYTHON3_FOR_TESTS NAMES python3 python) +if(PYTHON3_FOR_TESTS) + add_test(NAME tier_linkage + COMMAND ${PYTHON3_FOR_TESTS} ${CMAKE_CURRENT_SOURCE_DIR}/check_tier_linkage.py + $) +else() + message(WARNING "python3 not found, skipping the tier_linkage test") +endif() diff --git a/tests/unit/check_tier_linkage.py b/tests/unit/check_tier_linkage.py new file mode 100755 index 000000000..36d3ecb4e --- /dev/null +++ b/tests/unit/check_tier_linkage.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Guard the SIMD tier objects against sharing externally visible symbols. + +Every file under spaces/functions/ is compiled for one instruction-set tier, each with its +own -march/-m flags, and the running CPU's feature bits decide which tier's Choose_* entry +point is called. The kernels themselves are templates at namespace scope, so if two tiers +instantiate the same template they emit the same mangled symbol holding bodies built for +different architectures. The linker then keeps whichever object it saw first and nothing in +the source decides which. That either silently downgrades the faster tier or, once a body +contains an instruction the weaker tier's CPU lacks, faults in the query path. + +Tiers stay distinct by naming: a kernel reachable from two tiers must carry the tier in its +name, as NEON_HP/NEON_FHM and SVE/SVE2 do. This test checks that invariant on the built +archive, so a new tier or a newly shared kernel header cannot reintroduce the collision +unnoticed. + +Usage: check_tier_linkage.py +""" + +import re +import subprocess +import sys +from itertools import combinations + +# Object files under spaces/functions/, i.e. the per-tier translation units. Anything else in +# the archive (the dispatchers, the preprocessor container) is tier-neutral and shared on +# purpose, so it is not part of this invariant. +TIER_PREFIXES = ("NEON", "SVE", "AVX", "SSE", "F16C") + +# Tier-neutral helpers from shared type headers rather than from a kernel header. These are +# scalar bit manipulation with no instruction-set dependency, so every tier compiles them to +# the same bytes and link order cannot pick a wrong body. They surface only at -O0, where +# nothing is inlined away. The invariant this test enforces is about kernel code, so they are +# excluded by name rather than by weakening the check. +TIER_NEUTRAL = ("vecsim_types",) + +# Toolchain bookkeeping that can be emitted into every translation unit. These are not +# executable kernel code and cannot select a body built for the wrong ISA. +TOOLCHAIN_SYMBOLS = { + "___asan_globals_registered", + "DW.ref.__gxx_personality_v0", +} + + +def tier_symbols(archive): + """Map each tier object in the archive to its set of defined external symbols.""" + out = subprocess.run(["nm", "--defined-only", "--extern-only", archive], + check=True, capture_output=True, text=True).stdout + tiers, current = {}, None + for line in out.splitlines(): + member = re.fullmatch(r"(\S+\.o):", line.strip()) + if member: + name = member.group(1) + current = name if name.startswith(TIER_PREFIXES) else None + if current: + tiers.setdefault(current, set()) + continue + if current and len(line.split()) == 3: + symbol = line.split()[2] + if (symbol not in TOOLCHAIN_SYMBOLS and + not any(ns in symbol for ns in TIER_NEUTRAL)): + tiers[current].add(symbol) + return tiers + + +def main(): + if len(sys.argv) != 2: + sys.exit(__doc__) + archive = sys.argv[1] + tiers = tier_symbols(archive) + if not tiers: + sys.exit("no tier objects found in %s, so nothing was checked" % archive) + + failures = [] + for a, b in combinations(sorted(tiers), 2): + shared = tiers[a] & tiers[b] + if shared: + failures.append((a, b, sorted(shared))) + + print("checked %d tier objects, %d pairs" % (len(tiers), len(tiers) * (len(tiers) - 1) // 2)) + for name in sorted(tiers): + print(" %-28s %4d external symbols" % (name, len(tiers[name]))) + + if not failures: + print("PASS: no tier pair shares an externally visible symbol") + return + + for a, b, shared in failures: + print("\nFAIL: %s and %s both define %d symbol(s), so link order picks the body:" + % (a, b, len(shared))) + for sym in shared[:10]: + print(" %s" % sym) + if len(shared) > 10: + print(" ... and %d more" % (len(shared) - 10)) + sys.exit("\n%d tier pair(s) share symbols. Give the kernel a name carrying its tier, as " + "NEON_HP/NEON_FHM and SVE/SVE2 do." % len(failures)) + + +if __name__ == "__main__": + main() diff --git a/tests/unit/test_spaces.cpp b/tests/unit/test_spaces.cpp index c2aab0fd5..96456a9c5 100644 --- a/tests/unit/test_spaces.cpp +++ b/tests/unit/test_spaces.cpp @@ -43,6 +43,7 @@ #include "VecSim/spaces/functions/NEON.h" #include "VecSim/spaces/functions/NEON_DOTPROD.h" #include "VecSim/spaces/functions/NEON_HP.h" +#include "VecSim/spaces/functions/NEON_FHM.h" #include "VecSim/spaces/functions/NEON_BF16.h" #include "VecSim/spaces/functions/SVE.h" #include "VecSim/spaces/functions/SVE_BF16.h" @@ -3370,8 +3371,8 @@ TEST_P(SQ8_FP16_SpacesOptimizationTest, SQ8_FP16_L2SqrTest) { optimization.sve = 0; } #endif -#ifdef OPT_NEON_HP - if (optimization.asimdfhm) { +#ifdef OPT_NEON_FHM + if (optimization.asimdhp && optimization.asimdfhm) { unsigned char alignment = 0; arch_opt_func = L2_SQ8_FP16_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_FP16_L2_implementation_NEON_FHM(dim)) @@ -3381,6 +3382,8 @@ TEST_P(SQ8_FP16_SpacesOptimizationTest, SQ8_FP16_L2SqrTest) { ASSERT_EQ(alignment, 0) << "No alignment NEON_FHM with dim " << dim; optimization.asimdfhm = 0; } +#endif +#ifdef OPT_NEON_HP if (optimization.asimdhp) { unsigned char alignment = 0; arch_opt_func = L2_SQ8_FP16_GetDistFunc(dim, &alignment, &optimization); @@ -3494,8 +3497,8 @@ TEST_P(SQ8_FP16_SpacesOptimizationTest, SQ8_FP16_InnerProductTest) { optimization.sve = 0; } #endif -#ifdef OPT_NEON_HP - if (optimization.asimdfhm) { +#ifdef OPT_NEON_FHM + if (optimization.asimdhp && optimization.asimdfhm) { unsigned char alignment = 0; arch_opt_func = IP_SQ8_FP16_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_FP16_IP_implementation_NEON_FHM(dim)) @@ -3505,6 +3508,8 @@ TEST_P(SQ8_FP16_SpacesOptimizationTest, SQ8_FP16_InnerProductTest) { ASSERT_EQ(alignment, 0) << "No alignment NEON_FHM with dim " << dim; optimization.asimdfhm = 0; } +#endif +#ifdef OPT_NEON_HP if (optimization.asimdhp) { unsigned char alignment = 0; arch_opt_func = IP_SQ8_FP16_GetDistFunc(dim, &alignment, &optimization); @@ -3618,8 +3623,8 @@ TEST_P(SQ8_FP16_SpacesOptimizationTest, SQ8_FP16_CosineTest) { optimization.sve = 0; } #endif -#ifdef OPT_NEON_HP - if (optimization.asimdfhm) { +#ifdef OPT_NEON_FHM + if (optimization.asimdhp && optimization.asimdfhm) { unsigned char alignment = 0; arch_opt_func = Cosine_SQ8_FP16_GetDistFunc(dim, &alignment, &optimization); ASSERT_EQ(arch_opt_func, Choose_SQ8_FP16_Cosine_implementation_NEON_FHM(dim)) @@ -3629,6 +3634,8 @@ TEST_P(SQ8_FP16_SpacesOptimizationTest, SQ8_FP16_CosineTest) { ASSERT_EQ(alignment, 0) << "No alignment NEON_FHM with dim " << dim; optimization.asimdfhm = 0; } +#endif +#ifdef OPT_NEON_HP if (optimization.asimdhp) { unsigned char alignment = 0; arch_opt_func = Cosine_SQ8_FP16_GetDistFunc(dim, &alignment, &optimization);