Merge branch 'master' into bburkholder/indexwriter-closed-exception

slackhq · Feb 16, 2024 · 6411ca1 · 6411ca1
2 parents 46c47ea + 2c82356
commit 6411ca1
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 2 deletions.
diff --git a/kaldb/pom.xml b/kaldb/pom.xml
@@ -28,7 +28,7 @@
         <opensearch.version>2.11.1</opensearch.version>
         <curator.version>5.6.0</curator.version>
         <log4j.version>2.22.1</log4j.version>
-        <aws.sdk.version>2.22.9</aws.sdk.version>
+        <aws.sdk.version>2.24.4</aws.sdk.version>
         <error.prone.version>2.23.0</error.prone.version>
         <junit.jupiter.version>5.10.0</junit.jupiter.version>
     </properties>
@@ -332,7 +332,7 @@
         <dependency>
             <groupId>software.amazon.awssdk.crt</groupId>
             <artifactId>aws-crt</artifactId>
-            <version>0.29.4</version>
+            <version>0.29.10</version>
         </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>

diff --git a/kaldb/src/main/java/com/slack/kaldb/blobfs/s3/S3CrtBlobFs.java b/kaldb/src/main/java/com/slack/kaldb/blobfs/s3/S3CrtBlobFs.java
@@ -102,11 +102,23 @@ public static S3AsyncClient initS3Client(KaldbConfigs.S3Config config) {
         awsCredentialsProvider = DefaultCredentialsProvider.create();
       }
 
+      // default to 5% of the heap size for the max crt off-heap or 1GiB (min for client)
+      long jvmMaxHeapSizeBytes = Runtime.getRuntime().maxMemory();
+      long defaultCrtMemoryLimit = Math.max(Math.round(jvmMaxHeapSizeBytes * 0.05), 1073741824);
+      long maxNativeMemoryLimitBytes =
+          Long.parseLong(
+              System.getProperty(
+                  "kaldb.s3CrtBlobFs.maxNativeMemoryLimitBytes",
+                  String.valueOf(defaultCrtMemoryLimit)));
+      LOG.info(
+          "Using a maxNativeMemoryLimitInBytes for the S3AsyncClient of '{}' bytes",
+          maxNativeMemoryLimitBytes);
       S3CrtAsyncClientBuilder s3AsyncClient =
           S3AsyncClient.crtBuilder()
               .retryConfiguration(S3CrtRetryConfiguration.builder().numRetries(3).build())
               .targetThroughputInGbps(config.getS3TargetThroughputGbps())
               .region(Region.of(region))
+              .maxNativeMemoryLimitInBytes(maxNativeMemoryLimitBytes)
               .credentialsProvider(awsCredentialsProvider);
 
       // We add a healthcheck to prevent an error with the CRT client, where it will