Merge remote-tracking branch 'upstream/main' into fix_recover_hnsw

infiniflow · Jan 8, 2025 · e337a80 · e337a80
2 parents 4f922c2 + 5a1179a
commit e337a80
Show file tree

Hide file tree

Showing 689 changed files with 59,154 additions and 54,969 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -68,15 +68,25 @@ execute_process(
         OUTPUT_VARIABLE GIT_BRANCH_NAME
         OUTPUT_STRIP_TRAILING_WHITESPACE)
 execute_process(
-        COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD
-        OUTPUT_VARIABLE GIT_COMMIT_ID
+        COMMAND "${GIT_EXECUTABLE}" describe --tags --abbrev=0
+        OUTPUT_VARIABLE GIT_TAG
         OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(
+        COMMAND "${GIT_EXECUTABLE}" rev-list --count "${GIT_TAG}..HEAD"
+        OUTPUT_VARIABLE COMMIT_COUNT
+        OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(
+        COMMAND "${GIT_EXECUTABLE}" rev-parse --short HEAD
+        OUTPUT_VARIABLE HEAD_COMMIT_ID
+        OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+set(GIT_COMMIT_ID "${GIT_TAG}-${COMMIT_COUNT}-${HEAD_COMMIT_ID}")
 if("${GIT_BRANCH_NAME}" STREQUAL "")
     message(WARNING "Branch name not found.")
 else()
     message(STATUS "Branch name = ${GIT_BRANCH_NAME}")
 endif()
-if("${GIT_COMMIT_ID}" STREQUAL "")
+if("${HEAD_COMMIT_ID}" STREQUAL "")
     message(WARNING "Commit id not found.")
 else()
     message(STATUS "Commit-id = ${GIT_COMMIT_ID}")
@@ -260,7 +270,7 @@ if(DEFINED CPACK_PACKAGE_VERSION)
     string(REPLACE "-" "." CPACK_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION}")
 endif()
 if(NOT DEFINED CPACK_PACKAGE_VERSION OR CPACK_PACKAGE_VERSION STREQUAL "")
-    set(CPACK_PACKAGE_VERSION "0.5.2")
+    set(CPACK_PACKAGE_VERSION "${GIT_COMMIT_ID}")
 endif()
 set(CPACK_PACKAGE_RELEASE 1)
 set(CPACK_PACKAGE_CONTACT "Zhichang Yu <[email protected]>")

diff --git a/README.md b/README.md
@@ -82,6 +82,8 @@ Infinity supports two working modes, embedded mode and client-server mode. Infin
    print(res)
    ```
 
+> 💡 For more information about Infinity's Python API, see the [Python API Reference](https://infiniflow.org/docs/dev/pysdk_api_reference).
+
 #### 🔧 Deploy Infinity in client-server mode
 
 If you wish to deploy Infinity with the server and client as separate processes, see the [Deploy infinity server](https://infiniflow.org/docs/dev/deploy_infinity_server) guide.
@@ -90,8 +92,6 @@ If you wish to deploy Infinity with the server and client as separate processes,
 
 See the [Build from Source](https://infiniflow.org/docs/dev/build_from_source) guide.
 
-> 💡 For more information about Infinity's Python API, see the [Python API Reference](https://infiniflow.org/docs/dev/pysdk_api_reference).
-
 ## 📚 Document
 
 - [Quickstart](https://infiniflow.org/docs/dev/)
@@ -102,7 +102,7 @@ See the [Build from Source](https://infiniflow.org/docs/dev/build_from_source) g
 
 ## 📜 Roadmap
 
-See the [Infinity Roadmap 2024](https://github.com/infiniflow/infinity/issues/338)
+See the [Infinity Roadmap 2025](https://github.com/infiniflow/infinity/issues/2393)
 
 ## 🙌 Community
 

diff --git a/conf/infinity_conf.toml b/conf/infinity_conf.toml
@@ -43,6 +43,8 @@ mem_index_capacity       = 65536
 # secret_key               = "minioadmin"
 # enable_https             = false
 
+snapshot_dir            = "/var/infinity/snapshots"
+
 [buffer]
 buffer_manager_size      = "4GB"
 lru_num                  = 7

diff --git a/src/admin/admin_executor.cpp b/src/admin/admin_executor.cpp
@@ -3399,6 +3399,28 @@ QueryResult AdminExecutor::ListConfigs(QueryContext *query_context, const AdminS
         }
     }
 
+    {
+        {
+            // option name
+            Value value = Value::MakeVarchar(SNAPSHOT_DIR_OPTION_NAME);
+            ValueExpression value_expr(value);
+            value_expr.AppendToChunk(output_block_ptr->column_vectors[0]);
+        }
+        {
+            // option name type
+            Value value = Value::MakeVarchar(std::to_string(global_config->CleanupInterval()));
+            ValueExpression value_expr(value);
+            value_expr.AppendToChunk(output_block_ptr->column_vectors[1]);
+        }
+        {
+            // option name type
+            Value value = Value::MakeVarchar("Snapshots store directory");
+            ValueExpression value_expr(value);
+            value_expr.AppendToChunk(output_block_ptr->column_vectors[2]);
+        }
+    }
+
+
     {
         {
             // option name
@@ -3483,6 +3505,27 @@ QueryResult AdminExecutor::ListConfigs(QueryContext *query_context, const AdminS
         }
     }
 
+    {
+        {
+            // option name
+            Value value = Value::MakeVarchar(SNAPSHOT_DIR_OPTION_NAME);
+            ValueExpression value_expr(value);
+            value_expr.AppendToChunk(output_block_ptr->column_vectors[0]);
+        }
+        {
+            // option name type
+            Value value = Value::MakeVarchar(global_config->SnapshotDir());
+            ValueExpression value_expr(value);
+            value_expr.AppendToChunk(output_block_ptr->column_vectors[1]);
+        }
+        {
+            // option name type
+            Value value = Value::MakeVarchar("Snapshot storage directory");
+            ValueExpression value_expr(value);
+            value_expr.AppendToChunk(output_block_ptr->column_vectors[2]);
+        }
+    }
+
     {
         {
             // option name
@@ -4254,7 +4297,7 @@ QueryResult AdminExecutor::SetRole(QueryContext *query_context, const AdminState
 
             status = InfinityContext::instance().ChangeServerRole(NodeRole::kFollower, false, node_name, leader_ip, leader_port);
             if (!status.ok()) {
-                if(status.code() != ErrorCode::kCantSwitchRole) {
+                if (status.code() != ErrorCode::kCantSwitchRole) {
                     LOG_INFO("Fail to change to FOLLOWER role");
                     Status restore_status = InfinityContext::instance().ChangeServerRole(NodeRole::kAdmin);
                     if (!restore_status.ok()) {
@@ -4299,7 +4342,7 @@ QueryResult AdminExecutor::SetRole(QueryContext *query_context, const AdminState
 
             status = InfinityContext::instance().ChangeServerRole(NodeRole::kLearner, false, node_name, leader_ip, leader_port);
             if (!status.ok()) {
-                if(status.code() != ErrorCode::kCantSwitchRole) {
+                if (status.code() != ErrorCode::kCantSwitchRole) {
                     LOG_INFO("Fail to change to LEARNER role");
                     Status restore_status = InfinityContext::instance().ChangeServerRole(NodeRole::kAdmin);
                     if (!restore_status.ok()) {

diff --git a/src/admin/admin_executor.cppm b/src/admin/admin_executor.cppm
@@ -27,47 +27,48 @@ namespace infinity {
 
 export class AdminExecutor {
 public:
-    static QueryResult Execute(QueryContext* query_context, const AdminStatement* admin_statement);
+    static QueryResult Execute(QueryContext *query_context, const AdminStatement *admin_statement);
+
 private:
-    static QueryResult ListLogFiles(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ShowLogFile(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ListLogIndexes(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ShowLogIndex(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ListCatalogs(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ShowCatalog(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ListDatabases(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ShowDatabase(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ListTables(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ShowTable(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ListSegments(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ShowSegment(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ListBlocks(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ShowBlock(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ListColumns(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ShowColumn(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ListIndexes(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ShowIndex(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ListIndexSegments(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ShowIndexSegment(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ListConfigs(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ListVariables(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ShowVariable(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ListNodes(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ShowNode(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult RemoveNode(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ShowCurrentNode(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult SetRole(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult CreateSnapshot(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ListSnapshots(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ShowSnapshot(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult DeleteSnapshot(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult ExportSnapshot(QueryContext* query_context, const AdminStatement* admin_statement);
-    static QueryResult RecoverFromSnapshot(QueryContext* query_context, const AdminStatement* admin_statement);
+    static QueryResult ListLogFiles(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ShowLogFile(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ListLogIndexes(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ShowLogIndex(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ListCatalogs(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ShowCatalog(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ListDatabases(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ShowDatabase(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ListTables(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ShowTable(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ListSegments(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ShowSegment(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ListBlocks(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ShowBlock(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ListColumns(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ShowColumn(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ListIndexes(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ShowIndex(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ListIndexSegments(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ShowIndexSegment(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ListConfigs(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ListVariables(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ShowVariable(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ListNodes(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ShowNode(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult RemoveNode(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ShowCurrentNode(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult SetRole(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult CreateSnapshot(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ListSnapshots(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ShowSnapshot(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult DeleteSnapshot(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult ExportSnapshot(QueryContext *query_context, const AdminStatement *admin_statement);
+    static QueryResult RecoverFromSnapshot(QueryContext *query_context, const AdminStatement *admin_statement);
 
-    static Vector<SharedPtr<WalEntry>> GetAllCheckpointEntries(QueryContext* query_context, const AdminStatement* admin_statement);
-    static Vector<String> GetAllCheckpointFiles(QueryContext* query_context, const AdminStatement* admin_statement);
-    static Tuple<UniquePtr<Catalog>, Status> LoadCatalogFiles(QueryContext* query_context, const AdminStatement* admin_statement, Vector<SharedPtr<WalEntry>>& ckp_entries);
+    static Vector<SharedPtr<WalEntry>> GetAllCheckpointEntries(QueryContext *query_context, const AdminStatement *admin_statement);
+    static Vector<String> GetAllCheckpointFiles(QueryContext *query_context, const AdminStatement *admin_statement);
+    static Tuple<UniquePtr<Catalog>, Status>
+    LoadCatalogFiles(QueryContext *query_context, const AdminStatement *admin_statement, Vector<SharedPtr<WalEntry>> &ckp_entries);
 };
 
-}
-
+} // namespace infinity
diff --git a/src/bin/infinity_main.cpp b/src/bin/infinity_main.cpp
@@ -15,6 +15,7 @@
 #include <csignal>
 #include <cstdio>
 #include <cstdlib>
+#include <exception>
 #ifdef ENABLE_JEMALLOC_PROF
 #include <jemalloc/jemalloc.h>
 #endif
@@ -161,6 +162,7 @@ void SignalHandler(int signal_number, siginfo_t *, void *) {
         }
         case SIGSEGV: {
             // Print back strace
+            infinity::PrintTransactionHistory();
             infinity::PrintStacktrace("SEGMENT FAULTS");
             exit(-1);
             break;
@@ -196,6 +198,26 @@ void RegisterSignal() {
     sigaction(SIGSEGV, &sig_action, NULL);
 }
 
+void TerminateHandler() {
+    infinity::String message = "TerminateHandler: ";
+    try {
+        std::exception_ptr eptr{std::current_exception()};
+        if (eptr) {
+            std::rethrow_exception(eptr);
+        } else {
+            message += "Exiting without exception";
+        }
+    } catch (const std::exception &ex) {
+        message += "Unhandled Exception: ";
+        message += ex.what();
+    } catch (...) {
+        message += "Unknown Unhandled Exception";
+    }
+    infinity::PrintTransactionHistory();
+    infinity::PrintStacktrace(message);
+    std::abort();
+}
+
 } // namespace
 
 auto main(int argc, char **argv) -> int {
@@ -254,6 +276,8 @@ auto main(int argc, char **argv) -> int {
 
     RegisterSignal();
 
+    std::set_terminate(TerminateHandler);
+
     InfinityContext::instance().InitPhase2();
 
     shutdown_thread.join();

diff --git a/src/common/analyzer/analyzer_pool.cpp b/src/common/analyzer/analyzer_pool.cpp
@@ -38,14 +38,15 @@ import logger;
 
 namespace infinity {
 
-
 constexpr u64 basis = 0xCBF29CE484222325ull;
 constexpr u64 prime = 0x100000001B3ull;
 
 constexpr u64 Str2Int(const char *str, u64 last_value = basis) {
     return (*str != '\0' && *str != '-') ? Str2Int(str + 1, (*str ^ last_value) * prime) : last_value;
 }
 
+u64 AnalyzerPool::AnalyzerNameToInt(const char *str) { return Str2Int(str); }
+
 bool IcharEquals(char a, char b) { return ToLower(static_cast<int>(a)) == ToLower(static_cast<int>(b)); }
 
 bool IEquals(std::string_view lhs, std::string_view rhs) { return std::ranges::equal(lhs, rhs, IcharEquals); }
@@ -321,14 +322,16 @@ Tuple<UniquePtr<Analyzer>, Status> AnalyzerPool::GetAnalyzer(const std::string_v
             }
             return {MakeUnique<NGramAnalyzer>(ngram), Status::OK()};
         }
-        case Str2Int(KEYWORD.data()): {
-            return {MakeUnique<WhitespaceAnalyzer>(), Status::OK()};
-        }
+        case Str2Int(KEYWORD.data()):
         case Str2Int(WHITESPACE.data()): {
-            return {MakeUnique<WhitespaceAnalyzer>(), Status::OK()};
+            const auto suffix_pos = name.find_first_of('-');
+            if (suffix_pos == std::string_view::npos || suffix_pos + 1 == name.size()) {
+                return {MakeUnique<WhitespaceAnalyzer>(), Status::OK()};
+            }
+            return {MakeUnique<WhitespaceAnalyzer>(name.substr(suffix_pos + 1)), Status::OK()};
         }
         default: {
-            if(std::filesystem::is_regular_file(name)) {
+            if (std::filesystem::is_regular_file(name)) {
                 // Suppose it is a customized Python script analyzer
             }
             return {nullptr, Status::AnalyzerNotFound(name.data())};

diff --git a/src/common/analyzer/analyzer_pool.cppm b/src/common/analyzer/analyzer_pool.cppm
@@ -30,6 +30,8 @@ public:
 
     Tuple<UniquePtr<Analyzer>, Status> GetAnalyzer(const std::string_view &name);
 
+    static u64 AnalyzerNameToInt(const char *str);
+
     void Set(const std::string_view &name);
 
 public:

diff --git a/src/common/analyzer/jieba.cppm b/src/common/analyzer/jieba.cppm
@@ -22,9 +22,9 @@ module;
 
 export module jieba;
 
-export namespace cppjieba{
+export namespace cppjieba {
 
-    using cppjieba::Word;
-    using cppjieba::Jieba;
+using cppjieba::Jieba;
+using cppjieba::Word;
 
-}
+} // namespace cppjieba
diff --git a/src/common/analyzer/rag_analyzer.cpp b/src/common/analyzer/rag_analyzer.cpp
@@ -16,16 +16,16 @@ module;
 
 #define PCRE2_CODE_UNIT_WIDTH 8
 
+#include <cassert>
+#include <chrono>
 #include <cmath>
 #include <filesystem>
 #include <fstream>
+#include <iostream>
 #include <openccxx.h>
 #include <pcre2.h>
 #include <re2/re2.h>
 #include <sstream>
-#include <iostream>
-#include <chrono>
-#include <cassert>
 
 #include "string_utils.h"
 
@@ -768,7 +768,7 @@ int RAGAnalyzer::DFS(const String &chars,
     if (s >= len) {
         if (memo_all) {
             token_list.push_back(pre_tokens);
-        } else if (auto [vec_str ,current_score] = Score(pre_tokens); current_score > max_score) {
+        } else if (auto [vec_str, current_score] = Score(pre_tokens); current_score > max_score) {
             best_tokens = std::move(vec_str);
             max_score = current_score;
         }
@@ -1062,7 +1062,7 @@ inline void CheckDP2(const RAGAnalyzer *this_ptr, const std::string_view input_s
                                  std::format("[{} in top{}]", i + 1, topn));
     }
 }
-}
+} // namespace dp_debug
 #endif
 
 String RAGAnalyzer::Merge(const String &tks_str) const {
-Original file line number
+Diff line change
@@ Expand Up / @@ -30,6 +30,8 @@ public: @@
         Tuple<UniquePtr<Analyzer>, Status> GetAnalyzer(const std::string_view &name);
+        static u64 AnalyzerNameToInt(const char *str);
         void Set(const std::string_view &name);
     public:
@@ Expand Down @@