Support loading native lib directly from FS

staticlibs · staticlibs · commit d8dc9be4c298 · 2025-11-07T23:22:52.000Z
This PR is a continuation of duckdb#447 PR to allow using `duckdb_jdbc-x.x.x.x-nolib.jar` along with a JNI native library, that is loaded directly from file system. It extends the idea from duckdb#421 (and supersedes it) implementing the following logic: 1. if the driver JAR has a bundled native library (for current JVM os/arch), then this library will be unpacked to the temporary directory and loaded from there. If the library cannot be unpacked or loaded - there is no fallback to other methods (it is expected that `-nolib` JAR is used for other loading methods) 2. if the driver JAR does not hava a native library bundled inside it, then it will check whether a JNI native libary with the DuckDB internal naming (like `libduckdb_java.so_linux_amd64`) exists in file system next to the driver JAR (in the same directory). If library file is found there - then the driver will attempt to load it. If the file is found in file system, then it is expected that is can be loaded and there is no fallback to loading by name. 3. if the native lib is not found in the same directory, then, like in duckdb#421, the driver tries to load it using `duckdb_java` name (that will be translated by JVM to a platform-specific name like `libduckdb_java.so`). Testing: new test added that covers loading from the same dir and loading by name. Fixes: duckdb#444
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -560,10 +560,10 @@ file(GLOB_RECURSE JAVA_SRC_FILES src/main/java/org/duckdb/*.java)
 file(GLOB_RECURSE JAVA_TEST_FILES src/test/java/org/duckdb/*.java)
 set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g -Xlint:all)
 
-add_jar(duckdb_jdbc ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
+add_jar(duckdb_jdbc_nolib ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
         MANIFEST META-INF/MANIFEST.MF
         GENERATE_NATIVE_HEADERS duckdb-native)
-add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc)
+add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc_nolib)
 
 
 # main shared lib compilation
@@ -654,7 +654,10 @@ set_target_properties(duckdb_java PROPERTIES PREFIX "lib")
 
 add_custom_command(
   OUTPUT dummy_jdbc_target
-  DEPENDS duckdb_java duckdb_jdbc duckdb_jdbc_tests
+  DEPENDS duckdb_java duckdb_jdbc_nolib duckdb_jdbc_tests
+  COMMAND ${CMAKE_COMMAND} -E copy
+          duckdb_jdbc_nolib.jar
+          duckdb_jdbc.jar
   COMMAND ${Java_JAR_EXECUTABLE} uf duckdb_jdbc.jar -C
           $<TARGET_FILE_DIR:duckdb_java> $<TARGET_FILE_NAME:duckdb_java>)
 
diff --git a/CMakeLists.txt.in b/CMakeLists.txt.in
@@ -86,10 +86,10 @@ file(GLOB_RECURSE JAVA_SRC_FILES src/main/java/org/duckdb/*.java)
 file(GLOB_RECURSE JAVA_TEST_FILES src/test/java/org/duckdb/*.java)
 set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g -Xlint:all)
 
-add_jar(duckdb_jdbc ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
+add_jar(duckdb_jdbc_nolib ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
         MANIFEST META-INF/MANIFEST.MF
         GENERATE_NATIVE_HEADERS duckdb-native)
-add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc)
+add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc_nolib)
 
 
 # main shared lib compilation
@@ -180,7 +180,10 @@ set_target_properties(duckdb_java PROPERTIES PREFIX "lib")
 
 add_custom_command(
   OUTPUT dummy_jdbc_target
-  DEPENDS duckdb_java duckdb_jdbc duckdb_jdbc_tests
+  DEPENDS duckdb_java duckdb_jdbc_nolib duckdb_jdbc_tests
+  COMMAND ${CMAKE_COMMAND} -E copy
+          duckdb_jdbc_nolib.jar
+          duckdb_jdbc.jar
   COMMAND ${Java_JAR_EXECUTABLE} uf duckdb_jdbc.jar -C
           $<TARGET_FILE_DIR:duckdb_java> $<TARGET_FILE_NAME:duckdb_java>)
 
diff --git a/src/main/java/org/duckdb/DuckDBNative.java b/src/main/java/org/duckdb/DuckDBNative.java
@@ -1,66 +1,129 @@
 package org.duckdb;
 
-import java.io.File;
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.math.BigDecimal;
+import java.net.URI;
 import java.net.URL;
 import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
+import java.security.CodeSource;
+import java.security.ProtectionDomain;
 import java.sql.SQLException;
 import java.util.Properties;
 
 final class DuckDBNative {
+
+    private static final String ARCH_X86_64 = "amd64";
+    private static final String ARCH_AARCH64 = "arm64";
+    private static final String ARCH_UNIVERSAL = "universal";
+
+    private static final String OS_WINDOWS = "windows";
+    private static final String OS_MACOS = "osx";
+    private static final String OS_LINUX = "linux";
+
     static {
         try {
-            String os_name = "";
-            String os_arch;
-            String os_name_detect = System.getProperty("os.name").toLowerCase().trim();
-            String os_arch_detect = System.getProperty("os.arch").toLowerCase().trim();
-            switch (os_arch_detect) {
-            case "x86_64":
-            case "amd64":
-                os_arch = "amd64";
-                break;
-            case "aarch64":
-            case "arm64":
-                os_arch = "arm64";
-                break;
-            case "i386":
-                os_arch = "i386";
-                break;
-            default:
-                throw new IllegalStateException("Unsupported system architecture");
-            }
-            if (os_name_detect.startsWith("windows")) {
-                os_name = "windows";
-            } else if (os_name_detect.startsWith("mac")) {
-                os_name = "osx";
-                os_arch = "universal";
-            } else if (os_name_detect.startsWith("linux")) {
-                os_name = "linux";
-            }
-            String lib_res_name = "/libduckdb_java.so"
-                                  + "_" + os_name + "_" + os_arch;
-
-            Path lib_file = Files.createTempFile("libduckdb_java", ".so");
-            URL lib_res = DuckDBNative.class.getResource(lib_res_name);
-            if (lib_res == null) {
-                System.load(Paths.get("build/debug", lib_res_name).normalize().toAbsolutePath().toString());
-            } else {
-                try (final InputStream lib_res_input_stream = lib_res.openStream()) {
-                    Files.copy(lib_res_input_stream, lib_file, StandardCopyOption.REPLACE_EXISTING);
-                }
-                new File(lib_file.toString()).deleteOnExit();
-                System.load(lib_file.toAbsolutePath().toString());
-            }
-        } catch (IOException e) {
+            loadNativeLibrary();
+        } catch (Exception e) {
             throw new RuntimeException(e);
         }
     }
+
+    private static void loadNativeLibrary() throws Exception {
+        String libName = nativeLibName();
+        URL libRes = DuckDBNative.class.getResource("/" + libName);
+
+        // The current JAR has a native library bundled, in this case we unpack and load it.
+        // There is no fallback if the unpacking or loading fails. We expect that only
+        // the '-nolib' JAR can be used with external native lib.
+        if (null != libRes) {
+            unpackAndLoad(libRes);
+            return;
+        }
+
+        // External native library must be loaded, first we try to load it from the
+        // same directory where this JAR resides.
+        Path dir = currentJarDir();
+        if (null != dir) {
+            Path libPath = dir.resolve(libName);
+            if (Files.exists(libPath)) {
+                // Native lib is found next to this JAR, we expect it to be loadable
+                // and do NOT fall back to load-by-name if the loading fails.
+                System.load(libPath.toAbsolutePath().toString());
+                return;
+            }
+        }
+
+        // There is no native library next to the JAR file, so we try to load it by name.
+        System.loadLibrary("duckdb_java");
+    }
+
+    private static String cpuArch() throws SQLException {
+        String prop = System.getProperty("os.arch").toLowerCase().trim();
+        switch (prop) {
+        case "x86_64":
+        case "amd64":
+            return ARCH_X86_64;
+        case "aarch64":
+        case "arm64":
+            return ARCH_AARCH64;
+        default:
+            throw new SQLException("Unsupported system architecture: '" + prop + "'");
+        }
+    }
+
+    static String osName() throws SQLException {
+        String prop = System.getProperty("os.name").toLowerCase().trim();
+        if (prop.startsWith("windows")) {
+            return OS_WINDOWS;
+        } else if (prop.startsWith("mac")) {
+            return OS_MACOS;
+        } else if (prop.startsWith("linux")) {
+            return OS_LINUX;
+        } else {
+            throw new SQLException("Unsupported OS: '" + prop + "'");
+        }
+    }
+
+    static String nativeLibName() throws SQLException {
+        String os = osName();
+        final String arch;
+        if (OS_MACOS.equals(os)) {
+            arch = ARCH_UNIVERSAL;
+        } else {
+            arch = cpuArch();
+        }
+        return "libduckdb_java.so_" + os + "_" + arch;
+    }
+
+    static Path currentJarDir() {
+        try {
+            ProtectionDomain pd = DuckDBNative.class.getProtectionDomain();
+            CodeSource cs = pd.getCodeSource();
+            URL loc = cs.getLocation();
+            URI uri = loc.toURI();
+            Path jarPath = Paths.get(uri);
+            Path dirPath = jarPath.getParent();
+            return dirPath.toRealPath();
+        } catch (Exception e) {
+            return null;
+        }
+    }
+
+    private static void unpackAndLoad(URL nativeLibRes) throws IOException {
+        Path tmpFile = Files.createTempFile("libduckdb_java", ".so");
+        try (InputStream is = nativeLibRes.openStream()) {
+            Files.copy(is, tmpFile, REPLACE_EXISTING);
+        }
+        tmpFile.toFile().deleteOnExit();
+        System.load(tmpFile.toAbsolutePath().toString());
+    }
+
     // We use zero-length ByteBuffer-s as a hacky but cheap way to pass C++ pointers
     // back and forth
 
diff --git a/src/test/java/org/duckdb/TestDuckDBJDBC.java b/src/test/java/org/duckdb/TestDuckDBJDBC.java
@@ -3155,11 +3155,12 @@ public static void main(String[] args) throws Exception {
             Class<?> clazz = Class.forName("org.duckdb." + arg1);
             statusCode = runTests(new String[0], clazz);
         } else {
-            statusCode = runTests(args, TestDuckDBJDBC.class, TestAppender.class, TestAppenderCollection.class,
-                                  TestAppenderCollection2D.class, TestAppenderComposite.class,
-                                  TestSingleValueAppender.class, TestBatch.class, TestBindings.class, TestClosure.class,
-                                  TestExtensionTypes.class, TestSpatial.class, TestParameterMetadata.class,
-                                  TestPrepare.class, TestResults.class, TestSessionInit.class, TestTimestamp.class);
+            statusCode =
+                runTests(args, TestDuckDBJDBC.class, TestAppender.class, TestAppenderCollection.class,
+                         TestAppenderCollection2D.class, TestAppenderComposite.class, TestSingleValueAppender.class,
+                         TestBatch.class, TestBindings.class, TestClosure.class, TestExtensionTypes.class,
+                         TestNoLib.class, TestSpatial.class, TestParameterMetadata.class, TestPrepare.class,
+                         TestResults.class, TestSessionInit.class, TestTimestamp.class);
         }
         System.exit(statusCode);
     }
diff --git a/src/test/java/org/duckdb/TestNoLib.java b/src/test/java/org/duckdb/TestNoLib.java
@@ -0,0 +1,69 @@
+package org.duckdb;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.SQLException;
+
+public class TestNoLib {
+
+    private static Path javaExe() throws Exception {
+        String javaHomeProp = System.getProperty("java.home");
+        Path javaHome = Paths.get(javaHomeProp);
+        boolean isWindows = "windows".equals(DuckDBNative.osName());
+        return isWindows ? javaHome.resolve("bin/java.exe") : javaHome.resolve("bin/java");
+    }
+
+    private static void runQuickTest(Path currentJarDir) throws Exception {
+        String dir = currentJarDir.toAbsolutePath().toString();
+        ProcessBuilder pb = new ProcessBuilder(javaExe().toAbsolutePath().toString(), "-cp",
+                                               dir + File.separator + "duckdb_jdbc_tests.jar" + File.pathSeparator +
+                                                   dir + File.separator + "duckdb_jdbc_nolib.jar",
+                                               "org.duckdb.TestDuckDBJDBC", "test_spatial_POINT_2D")
+                                .inheritIO();
+        pb.environment().put("LD_LIBRARY_PATH", currentJarDir.toAbsolutePath().toString());
+        int code = pb.start().waitFor();
+        if (0 != code) {
+            throw new RuntimeException("Spawned test failure, code: " + code);
+        }
+    }
+
+    private static String platformLibName() throws Exception {
+        String os = DuckDBNative.osName();
+        switch (os) {
+        case "windows":
+            return "duckdb_java.dll";
+        case "osx":
+            return "libduckdb_java.dylib";
+        case "linux":
+            return "libduckdb_java.so";
+        default:
+            throw new SQLException("Unsupported OS: " + os);
+        }
+    }
+
+    public static void test_nolib() throws Exception {
+        Path dir = DuckDBNative.currentJarDir();
+        Path nativeLib = dir.resolve(DuckDBNative.nativeLibName());
+        Path platformNamedLib = dir.resolve(platformLibName());
+
+        // test next to JAR
+        if (Files.exists(platformNamedLib)) {
+            Files.delete(platformNamedLib);
+        }
+        System.out.println("\n----");
+        runQuickTest(dir);
+        System.out.println("----");
+
+        // test by name
+        try {
+            Files.move(nativeLib, platformNamedLib);
+            System.out.println("----");
+            runQuickTest(dir);
+            System.out.println("----");
+        } finally {
+            Files.move(platformNamedLib, nativeLib);
+        }
+    }
+}
diff --git a/src/test/java/org/duckdb/test/Runner.java b/src/test/java/org/duckdb/test/Runner.java
@@ -40,7 +40,7 @@ public static int runTests(String[] args, Class<?>... testClasses) {
         boolean anyFailed = false;
         for (Method m : methods) {
             if (m.getName().startsWith("test_")) {
-                if (quick_run && m.getName().startsWith("test_lots_")) {
+                if (quick_run && (m.getName().startsWith("test_lots_") || m.getName().startsWith("test_nolib"))) {
                     continue;
                 }
                 if (specific_test != null && !m.getName().contains(specific_test)) {

Original file line number	Diff line number	Diff line change
`@@ -40,7 +40,7 @@ public static int runTests(String[] args, Class<?>... testClasses) {`
`40`	`40`	`boolean anyFailed = false;`
`41`	`41`	`for (Method m : methods) {`
`42`	`42`	`if (m.getName().startsWith("test_")) {`
`43`		`- if (quick_run && m.getName().startsWith("test_lots_")) {`
	`43`	`+ if (quick_run && (m.getName().startsWith("test_lots_") \|\| m.getName().startsWith("test_nolib"))) {`
`44`	`44`	`continue;`
`45`	`45`	`}`
`46`	`46`	`if (specific_test != null && !m.getName().contains(specific_test)) {`