diff --git a/kaldb/pom.xml b/kaldb/pom.xml
index 709b02cdc4..c81ae59ece 100644
--- a/kaldb/pom.xml
+++ b/kaldb/pom.xml
@@ -28,7 +28,7 @@
2.11.1
5.6.0
2.22.1
- 2.22.9
+ 2.24.4
2.23.0
5.10.0
@@ -332,7 +332,7 @@
software.amazon.awssdk.crt
aws-crt
- 0.29.4
+ 0.29.10
software.amazon.awssdk
diff --git a/kaldb/src/main/java/com/slack/kaldb/blobfs/s3/S3CrtBlobFs.java b/kaldb/src/main/java/com/slack/kaldb/blobfs/s3/S3CrtBlobFs.java
index ad4ac7ee7e..49894706e6 100644
--- a/kaldb/src/main/java/com/slack/kaldb/blobfs/s3/S3CrtBlobFs.java
+++ b/kaldb/src/main/java/com/slack/kaldb/blobfs/s3/S3CrtBlobFs.java
@@ -102,11 +102,23 @@ public static S3AsyncClient initS3Client(KaldbConfigs.S3Config config) {
awsCredentialsProvider = DefaultCredentialsProvider.create();
}
+ // default to 5% of the heap size for the max crt off-heap or 1GiB (min for client)
+ long jvmMaxHeapSizeBytes = Runtime.getRuntime().maxMemory();
+ long defaultCrtMemoryLimit = Math.max(Math.round(jvmMaxHeapSizeBytes * 0.05), 1073741824);
+ long maxNativeMemoryLimitBytes =
+ Long.parseLong(
+ System.getProperty(
+ "kaldb.s3CrtBlobFs.maxNativeMemoryLimitBytes",
+ String.valueOf(defaultCrtMemoryLimit)));
+ LOG.info(
+ "Using a maxNativeMemoryLimitInBytes for the S3AsyncClient of '{}' bytes",
+ maxNativeMemoryLimitBytes);
S3CrtAsyncClientBuilder s3AsyncClient =
S3AsyncClient.crtBuilder()
.retryConfiguration(S3CrtRetryConfiguration.builder().numRetries(3).build())
.targetThroughputInGbps(config.getS3TargetThroughputGbps())
.region(Region.of(region))
+ .maxNativeMemoryLimitInBytes(maxNativeMemoryLimitBytes)
.credentialsProvider(awsCredentialsProvider);
// We add a healthcheck to prevent an error with the CRT client, where it will