diff --git a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala index 7c9d6ecb738..f58ac6ba155 100644 --- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala +++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala @@ -745,6 +745,7 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se def haMasterRatisStorageDir: String = get(HA_MASTER_RATIS_STORAGE_DIR) def haMasterRatisLogSegmentSizeMax: Long = get(HA_MASTER_RATIS_LOG_SEGMENT_SIZE_MAX) def haMasterRatisLogPreallocatedSize: Long = get(HA_MASTER_RATIS_LOG_PREALLOCATED_SIZE) + def haMasterRatisLogWriteBufferSize: Long = get(HA_MASTER_RATIS_LOG_WRITE_BUFFER_SIZE) def haMasterRatisLogAppenderQueueNumElements: Int = get(HA_MASTER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS) def haMasterRatisLogAppenderQueueBytesLimit: Long = @@ -2260,6 +2261,14 @@ object CelebornConf extends Logging { .bytesConf(ByteUnit.BYTE) .createWithDefaultString("4MB") + val HA_MASTER_RATIS_LOG_WRITE_BUFFER_SIZE: ConfigEntry[Long] = + buildConf("celeborn.master.ha.ratis.raft.server.log.write.buffer.size") + .internal + .categories("ha") + .version("0.5.0") + .bytesConf(ByteUnit.BYTE) + .createWithDefaultString("64MB") + val HA_MASTER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS: ConfigEntry[Int] = buildConf("celeborn.master.ha.ratis.raft.server.log.appender.buffer.element-limit") .withAlternative("celeborn.ha.master.ratis.raft.server.log.appender.buffer.element-limit") diff --git a/dev/deps/dependencies-client-flink-1.14 b/dev/deps/dependencies-client-flink-1.14 index 7a7f49b4a00..f3ce797acf2 100644 --- a/dev/deps/dependencies-client-flink-1.14 +++ b/dev/deps/dependencies-client-flink-1.14 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.18//scala-library-2.12.18.jar scala-reflect/2.12.18//scala-reflect-2.12.18.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-flink-1.15 b/dev/deps/dependencies-client-flink-1.15 index 7a7f49b4a00..f3ce797acf2 100644 --- a/dev/deps/dependencies-client-flink-1.15 +++ b/dev/deps/dependencies-client-flink-1.15 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.18//scala-library-2.12.18.jar scala-reflect/2.12.18//scala-reflect-2.12.18.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-flink-1.17 b/dev/deps/dependencies-client-flink-1.17 index 7a7f49b4a00..f3ce797acf2 100644 --- a/dev/deps/dependencies-client-flink-1.17 +++ b/dev/deps/dependencies-client-flink-1.17 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.18//scala-library-2.12.18.jar scala-reflect/2.12.18//scala-reflect-2.12.18.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-flink-1.18 b/dev/deps/dependencies-client-flink-1.18 index 7a7f49b4a00..f3ce797acf2 100644 --- a/dev/deps/dependencies-client-flink-1.18 +++ b/dev/deps/dependencies-client-flink-1.18 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.18//scala-library-2.12.18.jar scala-reflect/2.12.18//scala-reflect-2.12.18.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-flink-1.19 b/dev/deps/dependencies-client-flink-1.19 index 7a7f49b4a00..f3ce797acf2 100644 --- a/dev/deps/dependencies-client-flink-1.19 +++ b/dev/deps/dependencies-client-flink-1.19 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.18//scala-library-2.12.18.jar scala-reflect/2.12.18//scala-reflect-2.12.18.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-mr b/dev/deps/dependencies-client-mr index 309398c85f7..e952be60b22 100644 --- a/dev/deps/dependencies-client-mr +++ b/dev/deps/dependencies-client-mr @@ -180,10 +180,10 @@ okhttp/4.9.3//okhttp-4.9.3.jar okio/2.8.0//okio-2.8.0.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar re2j/1.1//re2j-1.1.jar reload4j/1.2.22//reload4j-1.2.22.jar scala-library/2.12.18//scala-library-2.12.18.jar diff --git a/dev/deps/dependencies-client-spark-2.4 b/dev/deps/dependencies-client-spark-2.4 index 5b0e11c527f..dc0d1a71891 100644 --- a/dev/deps/dependencies-client-spark-2.4 +++ b/dev/deps/dependencies-client-spark-2.4 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.11.12//scala-library-2.11.12.jar scala-reflect/2.11.12//scala-reflect-2.11.12.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-spark-3.0 b/dev/deps/dependencies-client-spark-3.0 index 856c67a1737..ffb7b0599d8 100644 --- a/dev/deps/dependencies-client-spark-3.0 +++ b/dev/deps/dependencies-client-spark-3.0 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.10//scala-library-2.12.10.jar scala-reflect/2.12.10//scala-reflect-2.12.10.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-spark-3.1 b/dev/deps/dependencies-client-spark-3.1 index a92623e1f78..381b67ff46d 100644 --- a/dev/deps/dependencies-client-spark-3.1 +++ b/dev/deps/dependencies-client-spark-3.1 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.10//scala-library-2.12.10.jar scala-reflect/2.12.10//scala-reflect-2.12.10.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-spark-3.2 b/dev/deps/dependencies-client-spark-3.2 index 3bf69bab048..ba87c7a60a8 100644 --- a/dev/deps/dependencies-client-spark-3.2 +++ b/dev/deps/dependencies-client-spark-3.2 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.15//scala-library-2.12.15.jar scala-reflect/2.12.15//scala-reflect-2.12.15.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-spark-3.3 b/dev/deps/dependencies-client-spark-3.3 index d647abdf13f..a5da10cf377 100644 --- a/dev/deps/dependencies-client-spark-3.3 +++ b/dev/deps/dependencies-client-spark-3.3 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.15//scala-library-2.12.15.jar scala-reflect/2.12.15//scala-reflect-2.12.15.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-spark-3.4 b/dev/deps/dependencies-client-spark-3.4 index aa4db0c9602..6f00059361d 100644 --- a/dev/deps/dependencies-client-spark-3.4 +++ b/dev/deps/dependencies-client-spark-3.4 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.17//scala-library-2.12.17.jar scala-reflect/2.12.17//scala-reflect-2.12.17.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-spark-3.5 b/dev/deps/dependencies-client-spark-3.5 index c93ed6efff1..bce13ca7c84 100644 --- a/dev/deps/dependencies-client-spark-3.5 +++ b/dev/deps/dependencies-client-spark-3.5 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.18//scala-library-2.12.18.jar scala-reflect/2.12.18//scala-reflect-2.12.18.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-server b/dev/deps/dependencies-server index c420faf9832..ca9a26f9452 100644 --- a/dev/deps/dependencies-server +++ b/dev/deps/dependencies-server @@ -117,16 +117,16 @@ netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-grpc/2.5.1//ratis-grpc-2.5.1.jar -ratis-metrics/2.5.1//ratis-metrics-2.5.1.jar -ratis-netty/2.5.1//ratis-netty-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-server-api/2.5.1//ratis-server-api-2.5.1.jar -ratis-server/2.5.1//ratis-server-2.5.1.jar -ratis-shell/2.5.1//ratis-shell-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-grpc/3.0.1//ratis-grpc-3.0.1.jar +ratis-metrics-default/3.0.1/ratis-metrics-default-3.0.1.jar +ratis-netty/3.0.1//ratis-netty-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-server-api/3.0.1//ratis-server-api-3.0.1.jar +ratis-server/3.0.1//ratis-server-3.0.1.jar +ratis-shell/3.0.1//ratis-shell-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar reflections/0.10.2//reflections-0.10.2.jar rocksdbjni/8.11.3//rocksdbjni-8.11.3.jar scala-library/2.12.18//scala-library-2.12.18.jar diff --git a/master/pom.xml b/master/pom.xml index 4dd3ed11c77..97ec5dd46fd 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -74,6 +74,10 @@ org.apache.ratis ratis-shell + + org.apache.ratis + ratis-metrics-default + org.apache.logging.log4j log4j-slf4j-impl diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java index b9d690d3372..7945410ff67 100644 --- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java +++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java @@ -17,6 +17,8 @@ package org.apache.celeborn.service.deploy.master.clustermeta.ha; +import static org.apache.ratis.server.storage.RaftStorage.StartupOption.RECOVER; + import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; @@ -129,6 +131,7 @@ private HARaftServer( .setGroup(this.raftGroup) .setProperties(serverProperties) .setStateMachine(masterStateMachine) + .setOption(RECOVER) // RATIS-1677. Do not auto format RaftStorage in RECOVER. .build(); StringBuilder raftPeersStr = new StringBuilder(); @@ -299,6 +302,7 @@ private RaftProperties newRaftProperties(CelebornConf conf) { // Set RAFT segment pre-allocated size long raftSegmentPreallocatedSize = conf.haMasterRatisLogPreallocatedSize(); + long raftSegmentWriteBufferSize = conf.haMasterRatisLogWriteBufferSize(); int logAppenderQueueNumElements = conf.haMasterRatisLogAppenderQueueNumElements(); long logAppenderQueueByteLimit = conf.haMasterRatisLogAppenderQueueBytesLimit(); boolean shouldInstallSnapshot = conf.haMasterRatisLogInstallSnapshotEnabled(); @@ -308,6 +312,9 @@ private RaftProperties newRaftProperties(CelebornConf conf) { properties, SizeInBytes.valueOf(logAppenderQueueByteLimit)); RaftServerConfigKeys.Log.setPreallocatedSize( properties, SizeInBytes.valueOf(raftSegmentPreallocatedSize)); + // RATIS-589. Eliminate buffer copying in SegmentedRaftLogOutputStream. + RaftServerConfigKeys.Log.setWriteBufferSize( + properties, SizeInBytes.valueOf(raftSegmentWriteBufferSize)); RaftServerConfigKeys.Log.Appender.setInstallSnapshotEnabled(properties, shouldInstallSnapshot); int logPurgeGap = conf.haMasterRatisLogPurgeGap(); RaftServerConfigKeys.Log.setPurgeGap(properties, logPurgeGap); diff --git a/pom.xml b/pom.xml index 357bb1839b4..d0df8d09444 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ 4.1.109.Final 1.77 3.21.7 - 2.5.1 + 3.0.1 3.2.16 1.7.36 1.0.6 @@ -280,6 +280,11 @@ + + org.apache.ratis + ratis-metrics-default + ${ratis.version} + org.scala-lang scala-library diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala index c5a928b0a06..558cfb54ba0 100644 --- a/project/CelebornBuild.scala +++ b/project/CelebornBuild.scala @@ -56,7 +56,7 @@ object Dependencies { val metricsVersion = "3.2.6" val mockitoVersion = "4.11.0" val nettyVersion = "4.1.109.Final" - val ratisVersion = "2.5.1" + val ratisVersion = "3.0.1" val roaringBitmapVersion = "1.0.6" val rocksdbJniVersion = "8.11.3" val jacksonVersion = "2.15.3" @@ -119,6 +119,7 @@ object Dependencies { val ratisClient = "org.apache.ratis" % "ratis-client" % ratisVersion val ratisCommon = "org.apache.ratis" % "ratis-common" % ratisVersion val ratisGrpc = "org.apache.ratis" % "ratis-grpc" % ratisVersion + val ratisMetrics = "org.apache.ratis" % "ratis-metrics-default" % ratisVersion val ratisNetty = "org.apache.ratis" % "ratis-netty" % ratisVersion val ratisServer = "org.apache.ratis" % "ratis-server" % ratisVersion val ratisShell = "org.apache.ratis" % "ratis-shell" % ratisVersion excludeAll( @@ -540,6 +541,7 @@ object CelebornMaster { Dependencies.ratisClient, Dependencies.ratisCommon, Dependencies.ratisGrpc, + Dependencies.ratisMetrics, Dependencies.ratisNetty, Dependencies.ratisServer, Dependencies.ratisShell