Support hybrid search, full-text search on tantivy inverted index

.gitmodules

@@ -338,3 +338,6 @@
 [submodule "contrib/search-index"]
  path = contrib/search-index
  url = https://github.com/myscale/search-index.git
+[submodule "rust/supercrate/libs/tantivy_search"]
+ path = rust/supercrate/libs/tantivy_search
+ url = [email protected]:mqdb/tantivy-search.git

cmake/dbms_glob_sources.cmake

@@ -11,3 +11,25 @@ endmacro()
 macro(add_headers_only prefix common_path)
  add_glob(${prefix}_headers ${CMAKE_CURRENT_SOURCE_DIR} ${common_path}/*.h)
 endmacro()
+
+function(remove_specific_headers_and_sources prefix)
+ set(files_to_remove ${ARGN})
+ set(root_path "${CMAKE_CURRENT_SOURCE_DIR}")
+
+ foreach(file IN LISTS files_to_remove)
+ get_filename_component(full_path "${file}" ABSOLUTE)
+ if("${full_path}" MATCHES "^${root_path}")
+ file(RELATIVE_PATH relative_path "${root_path}" "${full_path}")
+ set(file_to_remove "${relative_path}")
+ else()
+ set(file_to_remove "${file}")
+ endif()
+ # remove files from headers and sources
+ list(REMOVE_ITEM ${prefix}_headers "${file_to_remove}")
+ list(REMOVE_ITEM ${prefix}_sources "${file_to_remove}")
+ endforeach()
+
+ # update headers and sources
+ set(${prefix}_headers ${${prefix}_headers} PARENT_SCOPE)
+ set(${prefix}_sources ${${prefix}_sources} PARENT_SCOPE)
+endfunction()

contrib/CMakeLists.txt

@@ -94,7 +94,7 @@ add_contrib (protobuf-cmake protobuf)
 add_contrib (openldap-cmake openldap)
 add_contrib (grpc-cmake grpc)
 add_contrib (msgpack-c-cmake msgpack-c)
-
+# rust toolchain
 add_contrib (corrosion-cmake corrosion)
 
 if (ENABLE_FUZZING)

contrib/croaring-cmake/CMakeLists.txt

@@ -1,20 +1,22 @@
 set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/croaring")
 
 set(SRCS
- "${LIBRARY_DIR}/src/array_util.c"
- "${LIBRARY_DIR}/src/bitset_util.c"
  "${LIBRARY_DIR}/src/containers/array.c"
  "${LIBRARY_DIR}/src/containers/bitset.c"
  "${LIBRARY_DIR}/src/containers/containers.c"
  "${LIBRARY_DIR}/src/containers/convert.c"
- "${LIBRARY_DIR}/src/containers/mixed_intersection.c"
- "${LIBRARY_DIR}/src/containers/mixed_union.c"
+ "${LIBRARY_DIR}/src/containers/mixed_andnot.c"
  "${LIBRARY_DIR}/src/containers/mixed_equal.c"
- "${LIBRARY_DIR}/src/containers/mixed_subset.c"
+ "${LIBRARY_DIR}/src/containers/mixed_intersection.c"
  "${LIBRARY_DIR}/src/containers/mixed_negation.c"
+ "${LIBRARY_DIR}/src/containers/mixed_subset.c"
+ "${LIBRARY_DIR}/src/containers/mixed_union.c"
  "${LIBRARY_DIR}/src/containers/mixed_xor.c"
- "${LIBRARY_DIR}/src/containers/mixed_andnot.c"
  "${LIBRARY_DIR}/src/containers/run.c"
+ "${LIBRARY_DIR}/src/array_util.c"
+ "${LIBRARY_DIR}/src/bitset_util.c"
+ "${LIBRARY_DIR}/src/bitset.c"
+ "${LIBRARY_DIR}/src/isadetection.c"
  "${LIBRARY_DIR}/src/roaring.c"
  "${LIBRARY_DIR}/src/roaring_priority_queue.c"
  "${LIBRARY_DIR}/src/roaring_array.c"

programs/client/CMakeLists.txt

@@ -13,8 +13,9 @@ set (CLICKHOUSE_CLIENT_LINK
  string_utils
 )
 
-if (TARGET ch_rust::skim)
- list(APPEND CLICKHOUSE_CLIENT_LINK PRIVATE ch_rust::skim)
+# Need skim library in supercrate.
+if (TARGET ch_rust::supercrate)
+ list(APPEND CLICKHOUSE_CLIENT_LINK PRIVATE ch_rust::supercrate)
 endif()
 
 # Always use internal readpassphrase

programs/local/CMakeLists.txt

@@ -16,9 +16,11 @@ clickhouse_program_add(local)
 
 target_link_libraries(clickhouse-local-lib PRIVATE clickhouse-server-lib)
 
-if (TARGET ch_rust::skim)
- target_link_libraries(clickhouse-local-lib PRIVATE ch_rust::skim)
+# Need skim library in supercrate.
+if (TARGET ch_rust::supercrate)
+ target_link_libraries(clickhouse-local-lib PRIVATE ch_rust::supercrate)
 endif()
+
 if (TARGET ch_contrib::azure_sdk)
  target_link_libraries(clickhouse-local-lib PRIVATE ch_contrib::azure_sdk)
 endif()

programs/server/Server.cpp

@@ -98,6 +98,10 @@
 #include <VectorIndex/Common/VIBuildMemoryUsageHelper.h>
 #include <VectorIndex/Common/VICommon.h>
 
+#if USE_TANTIVY_SEARCH
+# include <tantivy_search.h>
+#endif
+
 #include "config.h"
 #include "config_version.h"
 
@@ -452,6 +456,50 @@ void setOOMScore(int value, Poco::Logger * log)
 }
 #endif
 
+extern "C" void tantivy_log_callback(int level, const char * thread_info, const char * message)
+{
+ Poco::Logger & logger = Poco::Logger::get("TantivyLibrary");
+ switch (level)
+ {
+ case -2: // -2 -> fatal
+ LOG_FATAL(&logger, "{} - {}", thread_info, message);
+ break;
+ case -1: // -1 -> error
+ LOG_ERROR(&logger, "{} - {}", thread_info, message);
+ break;
+ case 0: // 0 -> warning
+ LOG_WARNING(&logger, "{} - {}", thread_info, message);
+ break;
+ case 1: // 1 -> info
+ LOG_INFO(&logger, "{} - {}", thread_info, message);
+ break;
+ case 2: // 2 -> debug
+ LOG_DEBUG(&logger, "{} - {}", thread_info, message);
+ break;
+ case 3: // 3 -> tracing
+ LOG_TRACE(&logger, "{} - {}", thread_info, message);
+ break;
+ default:
+ LOG_DEBUG(&logger, "{} - {}", thread_info, message);
+ }
+}
+
+void tantivy_log_integration(Poco::Util::AbstractConfiguration & config)
+{
+ std::string tantivy_search_log_level = config.getString("logger.tantivy_search_log_level", config.getString("logger.level", "info"));
+
+#if USE_TANTIVY_SEARCH
+ tantivy_search_log4rs_initialize_with_callback(
+ "", // Path for storing the `tantivy-search` log file.
+ tantivy_search_log_level.c_str(), // Sets the log level for `tantivy-search`, defaulting to the same log level as ClickHouse.
+ false, // `tantivy-search` log content will be recorded to a specific log file.
+ false, // Flag to control whether `tantivy-search` log messages are displayed in the console/terminal.
+ true, // If true, logs are exclusively recorded for `tantivy_search` and not for its submodule libraries (e.g., tantivy).
+ tantivy_log_callback // Logging `tantivy_search` logs using POCO LOG.
+ );
+#endif
+}
+
 
 void Server::uninitialize()
 {
@@ -483,7 +531,9 @@ void Server::initialize(Poco::Util::Application & self)
 {
  BaseDaemon::initialize(self);
  logger().information("starting up");
-
+#if USE_TANTIVY_SEARCH
+ tantivy_log_integration(config());
+#endif
  LOG_INFO(&logger(), "OS name: {}, version: {}, architecture: {}",
  Poco::Environment::osName(),
  Poco::Environment::osVersion(),
@@ -1100,6 +1150,14 @@ try
  fs::create_directories(vector_index_cache_path);
  }
 
+ {
+#if USE_TANTIVY_SEARCH
+ std::string tantivy_index_cache_path = config().getString("tantivy_index_cache_path", path / "tantivy_index_cache/");
+ global_context->setTantivyIndexCachePath(tantivy_index_cache_path);
+ fs::create_directories(tantivy_index_cache_path);
+#endif
+ }
+
  /// top_level_domains_lists
  {
  const std::string & top_level_domains_path = config().getString("top_level_domains_path", path / "top_level_domains/");
@@ -1273,6 +1331,9 @@ try
  // in a lot of places. For now, disable updating log configuration without server restart.
  //setTextLog(global_context->getTextLog());
  updateLevels(*config, logger());
+#if USE_TANTIVY_SEARCH
+ tantivy_log_integration(*config);
+#endif
  global_context->setClustersConfig(config, has_zookeeper);
  global_context->setMacros(std::make_unique<Macros>(*config, "macros", log));
  global_context->setExternalAuthenticatorsConfig(*config);

rust/CMakeLists.txt

@@ -14,11 +14,30 @@ macro(configure_rustc)
  set(RUST_CFLAGS "${RUST_CFLAGS} --sysroot ${CMAKE_SYSROOT}")
  endif()
 
+ if (USE_MUSL)
+ set(RUST_CXXFLAGS "${RUST_CXXFLAGS} -D_LIBCPP_HAS_MUSL_LIBC=1")
+ endif ()
+
+ if(CCACHE_EXECUTABLE MATCHES "/sccache$")
+ message(STATUS "Using RUSTC_WRAPPER: ${CCACHE_EXECUTABLE}")
+ set(RUSTCWRAPPER "rustc-wrapper = \"${CCACHE_EXECUTABLE}\"")
+ else()
+ set(RUSTCWRAPPER "")
+ endif()
+
+ set(RUSTFLAGS "[]")
+ set(RUST_CARGO_BUILD_STD "")
+ # For more info: https://doc.rust-lang.org/beta/unstable-book/compiler-flags/sanitizer.html#memorysanitizer
+ if (SANITIZE STREQUAL "memory")
+ set(RUST_CARGO_BUILD_STD "build-std = [\"std\", \"panic_abort\", \"core\", \"alloc\"]")
+ set(RUSTFLAGS "[\"-Zsanitizer=memory\", \"-Zsanitizer-memory-track-origins\"]")
+ endif()
+
  message(STATUS "RUST_CFLAGS: ${RUST_CFLAGS}")
  message(STATUS "RUST_CXXFLAGS: ${RUST_CXXFLAGS}")
+ message(STATUS "RUSTFLAGS: ${RUSTFLAGS}")
+ message(STATUS "RUST_CARGO_BUILD_STD: ${RUST_CARGO_BUILD_STD}")
 
- # NOTE: requires RW access for the source dir
- configure_file("${CMAKE_CURRENT_SOURCE_DIR}/.cargo/config.toml.in" "${CMAKE_CURRENT_SOURCE_DIR}/.cargo/config.toml" @ONLY)
 endmacro()
 configure_rustc()
 
@@ -34,9 +53,20 @@ function(clickhouse_import_crate)
  else()
  set(CMAKE_CONFIGURATION_TYPES "${CMAKE_BUILD_TYPE};debug")
  endif()
- # NOTE: we may use LTO for rust too
 
- corrosion_import_crate(NO_STD ${ARGN})
+ if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
+ set(profile "")
+ else()
+ if (ENABLE_THINLTO)
+ set(profile "release-thinlto")
+ else()
+ set(profile "release")
+ endif()
+ endif()
+
+ # Note, here --offline is not used, since on CI vendor archive is used, and
+ # passing --offline here will be inconvenient for local development.
+ corrosion_import_crate(NO_STD ${ARGN} PROFILE ${profile})
 endfunction()
 
 # Add crate from the build directory.
@@ -47,7 +77,6 @@ endfunction()
 #
 # And to avoid overlaps different builds for one source directory, crate will
 # be copied from source directory to the binary directory.
-file(COPY ".cargo" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
 function(add_rust_subdirectory src)
  set(dst "${CMAKE_CURRENT_BINARY_DIR}/${src}")
  message(STATUS "Copy ${src} to ${dst}")
@@ -67,5 +96,6 @@ function(add_rust_subdirectory src)
  VERBATIM)
 endfunction()
 
-add_rust_subdirectory (BLAKE3)
-add_rust_subdirectory (skim)
+# supercrate combines all the Rust libraries into one static library, 
+# which can avoid symbol conflicts that occur when ClickHouse links multiple Rust static libraries.
+add_rust_subdirectory (supercrate)