dylib presto changes + unit test

prestodb · Jan 7, 2025 · aedfb72 · aedfb72
1 parent 942cbbc
commit aedfb72
Show file tree

Hide file tree

Showing 5 changed files with 39 additions and 1 deletion.
diff --git a/presto-native-execution/presto_cpp/main/CMakeLists.txt b/presto-native-execution/presto_cpp/main/CMakeLists.txt
@@ -76,7 +76,8 @@ target_link_libraries(
   ${FOLLY_WITH_DEPENDENCIES}
   ${GLOG}
   ${GFLAGS_LIBRARIES}
-  pthread)
+  pthread
+  velox_dynamic_function_loader)
 
 # Enabling Parquet causes build errors with missing symbols on MacOS. This is
 # likely due to a conflict between Arrow Thrift from velox_hive_connector and
@@ -89,6 +90,7 @@ set_property(TARGET presto_server_lib PROPERTY JOB_POOL_LINK
                                                presto_link_job_pool)
 
 add_executable(presto_server PrestoMain.cpp)
+target_link_options(presto_server BEFORE PUBLIC "-Wl,-export-dynamic")
 
 # Moving velox_hive_connector and velox_tpch_connector to presto_server_lib
 # results in multiple link errors similar to the one below only on GCC.

diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.cpp b/presto-native-execution/presto_cpp/main/PrestoServer.cpp
@@ -44,6 +44,7 @@
 #include "velox/common/base/StatsReporter.h"
 #include "velox/common/caching/CacheTTLController.h"
 #include "velox/common/caching/SsdCache.h"
+#include "velox/common/dynamic_registry/DynamicLibraryLoader.h"
 #include "velox/common/file/FileSystems.h"
 #include "velox/common/memory/MmapAllocator.h"
 #include "velox/common/memory/SharedArbitrator.h"
@@ -392,6 +393,7 @@ void PrestoServer::run() {
   registerRemoteFunctions();
   registerVectorSerdes();
   registerPrestoPlanNodeSerDe();
+  registerDynamicFunctions();
 
   const auto numExchangeHttpClientIoThreads = std::max<size_t>(
       systemConfig->exchangeHttpClientNumIoThreadsHwMultiplier() *
@@ -1593,5 +1595,28 @@ protocol::NodeStatus PrestoServer::fetchNodeStatus() {
 
   return nodeStatus;
 }
+void PrestoServer::registerDynamicFunctions() {
+  auto systemConfig = SystemConfig::instance();
+  if (!systemConfig->pluginDir().empty()) {
+    // if it is a valid directory, traverse and call dynamic function loader
+    const fs::path path(systemConfig->pluginDir());
+    PRESTO_STARTUP_LOG(INFO) << path;
+    std::error_code
+        ec; // For using the non-throwing overloads of functions below.
+    if (fs::is_directory(path, ec)) {
+      using recursiveDirectoryIterator =
+          std::filesystem::recursive_directory_iterator;
+      std::set<std::string> extensions{".so", ".dylib"};
+      for (const auto& dirEntry : recursiveDirectoryIterator(path)) {
+        // skip non shared library files and directories from loading
+        auto dirEntryPath = dirEntry.path();
+        if (!fs::is_directory(dirEntry, ec) &&
+            extensions.find(dirEntryPath.extension()) != extensions.end()) {
+          facebook::velox::loadDynamicLibrary(dirEntryPath.c_str());
+        }
+      }
+    }
+  }
+}
 
 } // namespace facebook::presto
diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.h b/presto-native-execution/presto_cpp/main/PrestoServer.h
@@ -190,6 +190,8 @@ class PrestoServer {
 
   VeloxPlanValidator* getVeloxPlanValidator();
 
+  virtual void registerDynamicFunctions();
+
   /// Invoked to get the list of filters passed to the http server.
   std::vector<std::unique_ptr<proxygen::RequestHandlerFactory>>
   getHttpServerFilters();

diff --git a/presto-native-execution/presto_cpp/main/common/Configs.cpp b/presto-native-execution/presto_cpp/main/common/Configs.cpp
@@ -240,6 +240,7 @@ SystemConfig::SystemConfig() {
           STR_PROP(kCacheVeloxTtlCheckInterval, "1h"),
           BOOL_PROP(kEnableRuntimeMetricsCollection, false),
           BOOL_PROP(kPlanValidatorFailOnNestedLoopJoin, false),
+          STR_PROP(kPluginDir, ""),
       };
 }
 
@@ -763,6 +764,10 @@ bool SystemConfig::enableRuntimeMetricsCollection() const {
   return optionalProperty<bool>(kEnableRuntimeMetricsCollection).value();
 }
 
+std::string SystemConfig::pluginDir() const {
+  return optionalProperty(kPluginDir).value();
+}
+
 NodeConfig::NodeConfig() {
   registeredProps_ =
       std::unordered_map<std::string, folly::Optional<std::string>>{

diff --git a/presto-native-execution/presto_cpp/main/common/Configs.h b/presto-native-execution/presto_cpp/main/common/Configs.h
@@ -648,6 +648,8 @@ class SystemConfig : public ConfigBase {
   static constexpr std::string_view kInternalCommunicationJwtExpirationSeconds{
       "internal-communication.jwt.expiration-seconds"};
 
+  /// Optional string containing the path to the plugin directory
+  static constexpr std::string_view kPluginDir{"plugin.dir"};
   /// Below are the Presto properties from config.properties that get converted
   /// to their velox counterparts in BaseVeloxQueryConfig and used solely from
   /// BaseVeloxQueryConfig.
@@ -899,6 +901,8 @@ class SystemConfig : public ConfigBase {
   bool enableRuntimeMetricsCollection() const;
 
   bool prestoNativeSidecar() const;
+
+  std::string pluginDir() const;
 };
 
 /// Provides access to node properties defined in node.properties file.