*: support vector search (#9486)

close #9032

*: support vector search

Co-authored-by: JaySon <tshent@qq.com>
Co-authored-by: JaySon-Huang <tshent@qq.com>

Author: Lloyd-Pottiger

.gitmodules

@@ -146,3 +146,12 @@
 [submodule "contrib/not_null"]
 	path = contrib/not_null
 	url = https://github.com/bitwizeshift/not_null.git
+[submodule "contrib/usearch"]
+	path = contrib/usearch
+	url = https://github.com/unum-cloud/usearch.git
+[submodule "contrib/simsimd"]
+	path = contrib/simsimd
+	url = https://github.com/ashvardanian/SimSIMD
+[submodule "contrib/highfive"]
+	path = contrib/highfive
+	url = https://github.com/BlueBrain/HighFive

cmake/cpu_features.cmake

@@ -95,7 +95,7 @@ elseif (ARCH_AMD64)
     # so we do not set the flags to avoid core dump in old machines
     option (TIFLASH_ENABLE_AVX_SUPPORT "Use AVX/AVX2 instructions on x86_64" ON)
     option (TIFLASH_ENABLE_AVX512_SUPPORT "Use AVX512 instructions on x86_64" ON)
-    
+
     # `haswell` was released since 2013 with cpu feature avx2, bmi2. It's a practical arch for optimizer
     option (TIFLASH_ENABLE_ARCH_HASWELL_SUPPORT "Use instructions based on architecture `haswell` on x86_64" ON)
 

contrib/CMakeLists.txt

@@ -198,3 +198,12 @@ add_subdirectory(aws-cmake)
 add_subdirectory(simdjson)
 
 add_subdirectory(fastpforlib)
+
+add_subdirectory(usearch-cmake)
+
+add_subdirectory(simsimd-cmake)
+
+if (ENABLE_TESTS AND NOT CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
+    add_subdirectory(hdf5-cmake)
+    add_subdirectory(highfive-cmake)
+endif ()

contrib/hdf5-cmake/.gitignore

@@ -0,0 +1 @@
+/download/*

contrib/hdf5-cmake/CMakeLists.txt

@@ -0,0 +1,41 @@
+include(ExternalProject)
+
+# hdf5 is too large. Instead of adding as a submodule, let's simply download from GitHub.
+ExternalProject_Add(hdf5-external
+    PREFIX          ${CMAKE_CURRENT_BINARY_DIR}
+    DOWNLOAD_DIR    ${TiFlash_SOURCE_DIR}/contrib/hdf5-cmake/download
+    URL             https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5_1.14.4.3.zip
+    URL_HASH        MD5=bc987d22e787290127aacd7b99b4f31e
+    CMAKE_ARGS
+        -DCMAKE_BUILD_TYPE=Release
+        -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
+        -DBUILD_STATIC_LIBS=ON
+        -DBUILD_SHARED_LIBS=OFF
+        -DBUILD_TESTING=OFF
+        -DHDF5_BUILD_HL_LIB=OFF
+        -DHDF5_BUILD_TOOLS=OFF
+        -DHDF5_BUILD_CPP_LIB=ON
+        -DHDF5_BUILD_EXAMPLES=OFF
+        -DHDF5_ENABLE_Z_LIB_SUPPORT=OFF
+        -DHDF5_ENABLE_SZIP_SUPPORT=OFF
+    BUILD_BYPRODUCTS           <INSTALL_DIR>/lib/${CMAKE_FIND_LIBRARY_PREFIXES}hdf5.a  # Workaround for Ninja
+    USES_TERMINAL_DOWNLOAD     TRUE
+    USES_TERMINAL_CONFIGURE    TRUE
+    USES_TERMINAL_BUILD        TRUE
+    USES_TERMINAL_INSTALL      TRUE
+    EXCLUDE_FROM_ALL           TRUE
+    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+)
+
+ExternalProject_Get_Property(hdf5-external INSTALL_DIR)
+
+add_library(tiflash_contrib::hdf5 STATIC IMPORTED GLOBAL)
+set_target_properties(tiflash_contrib::hdf5 PROPERTIES
+    IMPORTED_LOCATION       ${INSTALL_DIR}/lib/${CMAKE_FIND_LIBRARY_PREFIXES}hdf5.a
+)
+add_dependencies(tiflash_contrib::hdf5 hdf5-external)
+
+file(MAKE_DIRECTORY ${INSTALL_DIR}/include)
+target_include_directories(tiflash_contrib::hdf5 SYSTEM INTERFACE
+    ${INSTALL_DIR}/include
+)

contrib/highfive

@@ -0,0 +1,18 @@
+set(HIGHFIVE_PROJECT_DIR "${TiFlash_SOURCE_DIR}/contrib/highfive")
+set(HIGHFIVE_SOURCE_DIR "${HIGHFIVE_PROJECT_DIR}/include")
+
+if (NOT EXISTS "${HIGHFIVE_SOURCE_DIR}/highfive/highfive.hpp")
+    message (FATAL_ERROR "submodule contrib/highfive not found")
+endif()
+
+add_library(_highfive INTERFACE)
+
+target_include_directories(_highfive SYSTEM INTERFACE
+    ${HIGHFIVE_SOURCE_DIR}
+)
+
+target_link_libraries(_highfive INTERFACE
+    tiflash_contrib::hdf5
+)
+
+add_library(tiflash_contrib::highfive ALIAS _highfive)

contrib/highfive-cmake/CMakeLists.txt

@@ -0,0 +1,13 @@
+set(SIMSIMD_PROJECT_DIR "${TiFlash_SOURCE_DIR}/contrib/simsimd")
+set(SIMSIMD_SOURCE_DIR "${SIMSIMD_PROJECT_DIR}/include")
+
+add_library(_simsimd INTERFACE)
+
+if (NOT EXISTS "${SIMSIMD_SOURCE_DIR}/simsimd/simsimd.h")
+    message (FATAL_ERROR "submodule contrib/simsimd not found")
+endif()
+
+target_include_directories(_simsimd SYSTEM INTERFACE
+    ${SIMSIMD_SOURCE_DIR})
+
+add_library(tiflash_contrib::simsimd ALIAS _simsimd)