fix #387: fix tls block

src/plugin/cpp_plugin.cpp

@@ -33,6 +33,41 @@ CppPluginManagerImpl::CppPluginManagerImpl(LightningGraph* db, const std::string
 
 CppPluginManagerImpl::~CppPluginManagerImpl() {}
 
+void CppPluginManagerImpl::OpenDynamicLib(const PluginInfoBase* pinfo, DynamicLibinfo &dinfo) {
+    using namespace lgraph::dll;
+    const auto* info = dynamic_cast<const PluginInfo*>(pinfo);
+    dinfo.lib_handle = LoadDynamicLibrary(info->path);
+    if (!dinfo.lib_handle) {
+        auto errMsg = GetLastErrorMsg();
+        if (errMsg.find("cannot allocate memory in static TLS block") != std::string::npos) {
+            errMsg += ". Wait for other tasks to finish and try again";
+        }
+        throw InputError("Failed to load the DLL: " + errMsg);
+    }
+    if (info->has_func) {
+        dinfo.func = GetDllFunction<PluginFunc*>(dinfo.lib_handle, "Process");
+        if (!dinfo.func) {
+            UnloadDynamicLibrary(dinfo.lib_handle);
+            throw InputError("Failed to get Process() function in the DLL: " + GetLastErrorMsg());
+        }
+    } else {
+        dinfo.get_sig_spec = GetDllFunction<SignatureGetter*>(dinfo.lib_handle, "GetSignature");
+        dinfo.func_txn = GetDllFunction<PluginFuncInTxn*>(dinfo.lib_handle, "ProcessInTxn");
+        if (!dinfo.func_txn) {
+            UnloadDynamicLibrary(dinfo.lib_handle);
+            throw InputError(
+                "Failed to get ProcessInTxn() function in the DLL: "
+                + GetLastErrorMsg());
+        }
+    }
+}
+
+void CppPluginManagerImpl::CloseDynamicLib(DynamicLibinfo &dinfo) {
+    using namespace lgraph::dll;
+    if (!UnloadDynamicLibrary(dinfo.lib_handle))
+        throw InternalError("Failed to unload library.");
+}
+
 void CppPluginManagerImpl::DoCall(lgraph_api::Transaction* txn,
                                   const std::string& user,
                                   AccessControlledDB* db_with_access_control,
@@ -45,54 +80,66 @@ void CppPluginManagerImpl::DoCall(lgraph_api::Transaction* txn,
 
     // TODO: support in_process // NOLINT
     bool r = false;
-    const PluginInfo* info = dynamic_cast<const PluginInfo*>(pinfo);
-    if (info->func) {
-        PluginFunc* procedure = info->func;
-        lgraph_api::GraphDB db(db_with_access_control, info->read_only);
+    DynamicLibinfo info;
+    OpenDynamicLib(pinfo, info);
+    if (info.func) {
+        PluginFunc* procedure = info.func;
+        lgraph_api::GraphDB db(db_with_access_control, pinfo->read_only);
         r = procedure(db, request, output);
-    } else if (info->func_txn && txn != nullptr) {
-        PluginFuncInTxn * procedure = info->func_txn;
+    } else if (info.func_txn && txn != nullptr) {
+        PluginFuncInTxn * procedure = info.func_txn;
         r = procedure(*txn, request, output);
     }
+    CloseDynamicLib(info);
+
     if (!r) throw InputError(FMA_FMT("Plugin returned false. Output: {}.", output));
 }
 
 void CppPluginManagerImpl::LoadPlugin(const std::string& user, const std::string& name,
                                       PluginInfoBase* pinfo) {
     using namespace lgraph::dll;
     PluginInfo* info = dynamic_cast<PluginInfo*>(pinfo);
-    std::string path = GetPluginPath(name);
-    info->lib_handle = LoadDynamicLibrary(path);
-    if (!info->lib_handle) throw InputError("Failed to load the DLL: " + GetLastErrorMsg());
-    info->get_sig_spec = GetDllFunction<SignatureGetter*>(info->lib_handle, "GetSignature");
-    // it's ok for plugin which DOES NOT have `GetSignature` function.
-    // Plugins without `GetSignature` are not guaranteed to call safely in InQueryCall context.
-    if (!info->get_sig_spec) {
-        info->func = GetDllFunction<PluginFunc*>(info->lib_handle, "Process");
-        if (!info->func) {
-            UnloadDynamicLibrary(info->lib_handle);
-            throw InputError("Failed to get Process() function in the DLL: " + GetLastErrorMsg());
+    info->path = GetPluginPath(name);
+    auto lib_handle = LoadDynamicLibrary(info->path);
+    if (!lib_handle) {
+        auto errMsg = GetLastErrorMsg();
+        if (errMsg.find("cannot allocate memory in static TLS block") != std::string::npos) {
+            errMsg += ". Wait for other tasks to finish and try again";
         }
-        info->sig_spec = nullptr;
-        return;
-    } else {
-        info->func_txn = GetDllFunction<PluginFuncInTxn*>(info->lib_handle, "ProcessInTxn");
-        if (!info->func_txn) {
-            UnloadDynamicLibrary(info->lib_handle);
-            throw InputError("Failed to get Process() function in the DLL: " + GetLastErrorMsg());
+        throw InputError("Failed to load the DLL: " + errMsg);
+    }
+    {
+        auto get_sig_spec = GetDllFunction<SignatureGetter*>(lib_handle, "GetSignature");
+        // it's ok for plugin which DOES NOT have `GetSignature` function.
+        // Plugins without `GetSignature` are not guaranteed to call safely in InQueryCall context.
+        if (!get_sig_spec) {
+            auto func = GetDllFunction<PluginFunc*>(lib_handle, "Process");
+            if (!func) {
+                UnloadDynamicLibrary(lib_handle);
+                throw InputError("Failed to get Process() function in the DLL: " +
+                                 GetLastErrorMsg());
+            }
+            info->sig_spec = nullptr;
+            info->has_func = true;
+        } else {
+            auto func_txn = GetDllFunction<PluginFuncInTxn*>(lib_handle, "ProcessInTxn");
+            if (!func_txn) {
+                UnloadDynamicLibrary(lib_handle);
+                throw InputError("Failed to get Process() function in the DLL: " +
+                                 GetLastErrorMsg());
+            }
+            auto sig_spec = std::make_unique<lgraph_api::SigSpec>();
+            bool r = get_sig_spec(*sig_spec);
+            if (!r) throw InputError(FMA_FMT("Failed to get Signature"));
+            info->sig_spec = std::move(sig_spec);
+            info->has_func = false;
         }
-        auto sig_spec = std::make_unique<lgraph_api::SigSpec>();
-        bool r = info->get_sig_spec(*sig_spec);
-        if (!r) throw InputError(FMA_FMT("Failed to get Signature"));
-        info->sig_spec = std::move(sig_spec);
+    }
+    if (!UnloadDynamicLibrary(lib_handle)) {
+        throw InputError("Failed to unload DLL: " + GetLastErrorMsg());
     }
 }
 
 void CppPluginManagerImpl::UnloadPlugin(const std::string& user, const std::string& name,
-                                        PluginInfoBase* pinfo) {
-    using namespace lgraph::dll;
-    PluginInfo* info = dynamic_cast<PluginInfo*>(pinfo);
-    if (!UnloadDynamicLibrary(info->lib_handle))
-        throw InternalError("Failed to unload library [{}].", name);
-}
+                                        PluginInfoBase* pinfo) {}
 }  // namespace lgraph

src/plugin/cpp_plugin.h

@@ -29,16 +29,48 @@ namespace lgraph {
 class LightningGraph;
 
 class CppPluginManagerImpl : public PluginManagerImplBase {
+ private:
+    struct DynamicLibinfo {
+        lgraph::dll::LibHandle lib_handle;
+        lgraph_api::Process* func = nullptr;
+        lgraph_api::ProcessInTxn* func_txn = nullptr;
+        lgraph_api::GetSignature* get_sig_spec = nullptr;
+    };
+    /**
+     * Open the dynamic library and find the run-time address
+     * refers to of the symbol "Process".
+     *
+     * Opening multiple algorithm libraries at the same time will cause the problem
+     * of "cannot allocate memory in static TLS block". Use OpenDynamicLib to load the
+     * plugin each time before calling the plugin, and use CloseDynamicLib to unload
+     * the plugin after calling the plugin.
+     *
+     * @param [in]      pinfo    If non-null, the pinfo.
+     * @param [in,out]  dinfo    plugin handle.
+     *
+     * */
+    void OpenDynamicLib(const PluginInfoBase* pinfo, DynamicLibinfo &dinfo);
+
+    /**
+     * Close the dynamic library.
+     *
+     * Opening multiple algorithm libraries at the same time will cause the problem
+     * of "cannot allocate memory in static TLS block". Use OpenDynamicLib to load the
+     * plugin each time before calling the plugin, and use CloseDynamicLib to unload
+     * the plugin after calling the plugin.
+     *
+     * @param [in,out]  dinfo    plugin handle.
+     * */
+    void CloseDynamicLib(DynamicLibinfo &dinfo);
+
  protected:
     typedef lgraph_api::Process PluginFunc;
     typedef lgraph_api::ProcessInTxn PluginFuncInTxn;
     typedef lgraph_api::GetSignature SignatureGetter;
 
     struct PluginInfo : public PluginInfoBase {
-        lgraph::dll::LibHandle lib_handle;
-        PluginFunc* func = nullptr;
-        PluginFuncInTxn* func_txn = nullptr;
-        SignatureGetter* get_sig_spec = nullptr;
+        std::string path;
+        bool has_func = true;
     };
 
     LightningGraph* db_;
@@ -55,6 +87,11 @@ class CppPluginManagerImpl : public PluginManagerImplBase {
     /**
      * Loads a plugin and sets contents in pinfo accordingly.
      *
+     * Opening multiple algorithm libraries at the same time will cause the problem
+     * of "cannot allocate memory in static TLS block". Call OpenDynamicLib to verify
+     * that there are no errors in the plugin file and then call CloseDynamicLib to
+     * uninstall the dynamic library.
+     *
      * @param          name     The name.
      * @param [in,out] pinfo    If non-null, the pinfo.
      * @param [in,out] error    The error.
@@ -67,6 +104,9 @@ class CppPluginManagerImpl : public PluginManagerImplBase {
     /**
      * Unload plugin and set pinfo if necessary.
      *
+     * Does nothing because the dynamic library has already been unloaded
+     * in CloseDynamicLib.
+     *
      * @param [in,out] pinfo    If non-null, the pinfo.
      * @param [in,out] error    The error.
      *

test/test_cpp_procedure.cpp

@@ -90,6 +90,10 @@ void build_so() {
                  "../../test/test_procedures/add_label_v.cpp", LIBLGRAPH);
     rt = system(cmd.c_str());
     UT_EXPECT_EQ(rt, 0);
+    cmd = UT_FMT(cmd_f.c_str(), INCLUDE_DIR, DEPS_INCLUDE_DIR, "./bfs.so",
+                 "../../test/test_procedures/bfs.cpp", LIBLGRAPH);
+    rt = system(cmd.c_str());
+    UT_EXPECT_EQ(rt, 0);
 }
 
 void read_code(const std::string& code_path, std::string& code) {
@@ -211,6 +215,7 @@ TEST_F(TestCppPlugin, CppPlugin) {
     std::string code_zip = "";
     std::string code_scan_graph = "";
     std::string code_add_label = "";
+    std::string code_bfs = "";
     {
         // read file to string
         std::string code_so_path = "./sortstr.so";
@@ -224,12 +229,14 @@ TEST_F(TestCppPlugin, CppPlugin) {
 #endif
         std::string code_scan_graph_path = "./scan_graph.so";
         std::string code_add_label_path = "./add_label.so";
+        std::string code_bfs_path = "./bfs.so";
         build_so();
         read_code(code_so_path, code_so);
         read_code(code_cpp_path, code_cpp);
         read_code(code_zip_path, code_zip);
         read_code(code_scan_graph_path, code_scan_graph);
         read_code(code_add_label_path, code_add_label);
+        read_code(code_bfs_path, code_bfs);
         UT_EXPECT_NE(code_so, "");
         UT_EXPECT_NE(code_cpp, "");
         UT_EXPECT_NE(code_zip, "");
@@ -445,6 +452,23 @@ TEST_F(TestCppPlugin, CppPlugin) {
                                                       "#include <stdlib.h>", (plugin::CodeType)6,
                                                       "test", true, "v1"));
         }
+#ifndef __SANITIZE_ADDRESS__
+        {
+            UT_LOG() << "Testing load many plugins";
+            for (int i = 0; i < 32; i++) {
+                UT_LOG() << "try load bfs_" << i;
+                bool r = false;
+                UT_EXPECT_NO_THROW(
+                    r = pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME,
+                         "bfs_" + std::to_string(i),
+                         code_bfs, plugin::CodeType::SO,
+                         "bfs v1", true, "v1"));
+                UT_EXPECT_TRUE(r);
+                fma_common::SleepS(5);
+            }
+            pm.DeleteAllPlugins(lgraph::_detail::DEFAULT_ADMIN_NAME);
+        }
+#endif  // __SANITIZE_ADDRESS__
     }
 #endif
 }