From 5df6bdcfe526dd04df19092f842c829b902a0617 Mon Sep 17 00:00:00 2001 From: SteNicholas Date: Wed, 24 Apr 2024 20:26:39 +0800 Subject: [PATCH] [CELEBORN-1400] Bump Ratis version from 2.5.1 to 3.0.1 --- .../apache/celeborn/common/CelebornConf.scala | 20 +++++++++++++++++++ dev/deps/dependencies-client-flink-1.14 | 8 ++++---- dev/deps/dependencies-client-flink-1.15 | 8 ++++---- dev/deps/dependencies-client-flink-1.17 | 8 ++++---- dev/deps/dependencies-client-flink-1.18 | 8 ++++---- dev/deps/dependencies-client-flink-1.19 | 8 ++++---- dev/deps/dependencies-client-mr | 8 ++++---- dev/deps/dependencies-client-spark-2.4 | 8 ++++---- dev/deps/dependencies-client-spark-3.0 | 8 ++++---- dev/deps/dependencies-client-spark-3.1 | 8 ++++---- dev/deps/dependencies-client-spark-3.2 | 8 ++++---- dev/deps/dependencies-client-spark-3.3 | 8 ++++---- dev/deps/dependencies-client-spark-3.4 | 8 ++++---- dev/deps/dependencies-client-spark-3.5 | 8 ++++---- dev/deps/dependencies-server | 20 +++++++++---------- docs/configuration/ha.md | 3 ++- master/pom.xml | 4 ++++ .../master/clustermeta/ha/HARaftServer.java | 19 ++++++++++++++++++ pom.xml | 7 ++++++- project/CelebornBuild.scala | 4 +++- 20 files changed, 116 insertions(+), 65 deletions(-) diff --git a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala index 68fa938b561..34ff7df8790 100644 --- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala +++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala @@ -743,8 +743,10 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se def haMasterRatisRpcType: String = get(HA_MASTER_RATIS_RPC_TYPE) def haMasterRatisStorageDir: String = get(HA_MASTER_RATIS_STORAGE_DIR) + def haMasterRatisStorageStartupOption: String = get(HA_MASTER_RATIS_STORAGE_STARTUP_OPTION) def haMasterRatisLogSegmentSizeMax: Long = get(HA_MASTER_RATIS_LOG_SEGMENT_SIZE_MAX) def haMasterRatisLogPreallocatedSize: Long = get(HA_MASTER_RATIS_LOG_PREALLOCATED_SIZE) + def haMasterRatisLogWriteBufferSize: Long = get(HA_MASTER_RATIS_LOG_WRITE_BUFFER_SIZE) def haMasterRatisLogAppenderQueueNumElements: Int = get(HA_MASTER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS) def haMasterRatisLogAppenderQueueBytesLimit: Long = @@ -2265,10 +2267,20 @@ object CelebornConf extends Logging { buildConf("celeborn.master.ha.ratis.raft.server.storage.dir") .withAlternative("celeborn.ha.master.ratis.raft.server.storage.dir") .categories("ha") + .doc("Root storage directory to hold RaftServer data.") .version("0.3.0") .stringConf .createWithDefault("/tmp/ratis") + val HA_MASTER_RATIS_STORAGE_STARTUP_OPTION: ConfigEntry[String] = + buildConf("celeborn.master.ha.ratis.raft.server.storage.startup.option") + .categories("ha") + .doc("Startup option of RaftServer storage. Available options: RECOVER, FORMAT.") + .version("0.5.0") + .stringConf + .checkValues(Set("RECOVER", "FORMAT")) + .createWithDefault("RECOVER") + val HA_MASTER_RATIS_LOG_SEGMENT_SIZE_MAX: ConfigEntry[Long] = buildConf("celeborn.master.ha.ratis.raft.server.log.segment.size.max") .withAlternative("celeborn.ha.master.ratis.raft.server.log.segment.size.max") @@ -2287,6 +2299,14 @@ object CelebornConf extends Logging { .bytesConf(ByteUnit.BYTE) .createWithDefaultString("4MB") + val HA_MASTER_RATIS_LOG_WRITE_BUFFER_SIZE: ConfigEntry[Long] = + buildConf("celeborn.master.ha.ratis.raft.server.log.write.buffer.size") + .internal + .categories("ha") + .version("0.5.0") + .bytesConf(ByteUnit.BYTE) + .createWithDefaultString("36MB") + val HA_MASTER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS: ConfigEntry[Int] = buildConf("celeborn.master.ha.ratis.raft.server.log.appender.buffer.element-limit") .withAlternative("celeborn.ha.master.ratis.raft.server.log.appender.buffer.element-limit") diff --git a/dev/deps/dependencies-client-flink-1.14 b/dev/deps/dependencies-client-flink-1.14 index 7a7f49b4a00..f3ce797acf2 100644 --- a/dev/deps/dependencies-client-flink-1.14 +++ b/dev/deps/dependencies-client-flink-1.14 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.18//scala-library-2.12.18.jar scala-reflect/2.12.18//scala-reflect-2.12.18.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-flink-1.15 b/dev/deps/dependencies-client-flink-1.15 index 7a7f49b4a00..f3ce797acf2 100644 --- a/dev/deps/dependencies-client-flink-1.15 +++ b/dev/deps/dependencies-client-flink-1.15 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.18//scala-library-2.12.18.jar scala-reflect/2.12.18//scala-reflect-2.12.18.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-flink-1.17 b/dev/deps/dependencies-client-flink-1.17 index 7a7f49b4a00..f3ce797acf2 100644 --- a/dev/deps/dependencies-client-flink-1.17 +++ b/dev/deps/dependencies-client-flink-1.17 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.18//scala-library-2.12.18.jar scala-reflect/2.12.18//scala-reflect-2.12.18.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-flink-1.18 b/dev/deps/dependencies-client-flink-1.18 index 7a7f49b4a00..f3ce797acf2 100644 --- a/dev/deps/dependencies-client-flink-1.18 +++ b/dev/deps/dependencies-client-flink-1.18 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.18//scala-library-2.12.18.jar scala-reflect/2.12.18//scala-reflect-2.12.18.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-flink-1.19 b/dev/deps/dependencies-client-flink-1.19 index 7a7f49b4a00..f3ce797acf2 100644 --- a/dev/deps/dependencies-client-flink-1.19 +++ b/dev/deps/dependencies-client-flink-1.19 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.18//scala-library-2.12.18.jar scala-reflect/2.12.18//scala-reflect-2.12.18.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-mr b/dev/deps/dependencies-client-mr index 309398c85f7..e952be60b22 100644 --- a/dev/deps/dependencies-client-mr +++ b/dev/deps/dependencies-client-mr @@ -180,10 +180,10 @@ okhttp/4.9.3//okhttp-4.9.3.jar okio/2.8.0//okio-2.8.0.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar re2j/1.1//re2j-1.1.jar reload4j/1.2.22//reload4j-1.2.22.jar scala-library/2.12.18//scala-library-2.12.18.jar diff --git a/dev/deps/dependencies-client-spark-2.4 b/dev/deps/dependencies-client-spark-2.4 index 5b0e11c527f..dc0d1a71891 100644 --- a/dev/deps/dependencies-client-spark-2.4 +++ b/dev/deps/dependencies-client-spark-2.4 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.11.12//scala-library-2.11.12.jar scala-reflect/2.11.12//scala-reflect-2.11.12.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-spark-3.0 b/dev/deps/dependencies-client-spark-3.0 index 856c67a1737..ffb7b0599d8 100644 --- a/dev/deps/dependencies-client-spark-3.0 +++ b/dev/deps/dependencies-client-spark-3.0 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.10//scala-library-2.12.10.jar scala-reflect/2.12.10//scala-reflect-2.12.10.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-spark-3.1 b/dev/deps/dependencies-client-spark-3.1 index a92623e1f78..381b67ff46d 100644 --- a/dev/deps/dependencies-client-spark-3.1 +++ b/dev/deps/dependencies-client-spark-3.1 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.10//scala-library-2.12.10.jar scala-reflect/2.12.10//scala-reflect-2.12.10.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-spark-3.2 b/dev/deps/dependencies-client-spark-3.2 index 3bf69bab048..ba87c7a60a8 100644 --- a/dev/deps/dependencies-client-spark-3.2 +++ b/dev/deps/dependencies-client-spark-3.2 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.15//scala-library-2.12.15.jar scala-reflect/2.12.15//scala-reflect-2.12.15.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-spark-3.3 b/dev/deps/dependencies-client-spark-3.3 index d647abdf13f..a5da10cf377 100644 --- a/dev/deps/dependencies-client-spark-3.3 +++ b/dev/deps/dependencies-client-spark-3.3 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.15//scala-library-2.12.15.jar scala-reflect/2.12.15//scala-reflect-2.12.15.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-spark-3.4 b/dev/deps/dependencies-client-spark-3.4 index aa4db0c9602..6f00059361d 100644 --- a/dev/deps/dependencies-client-spark-3.4 +++ b/dev/deps/dependencies-client-spark-3.4 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.17//scala-library-2.12.17.jar scala-reflect/2.12.17//scala-reflect-2.12.17.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-client-spark-3.5 b/dev/deps/dependencies-client-spark-3.5 index c93ed6efff1..bce13ca7c84 100644 --- a/dev/deps/dependencies-client-spark-3.5 +++ b/dev/deps/dependencies-client-spark-3.5 @@ -73,10 +73,10 @@ netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar scala-library/2.12.18//scala-library-2.12.18.jar scala-reflect/2.12.18//scala-reflect-2.12.18.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar diff --git a/dev/deps/dependencies-server b/dev/deps/dependencies-server index c420faf9832..ca9a26f9452 100644 --- a/dev/deps/dependencies-server +++ b/dev/deps/dependencies-server @@ -117,16 +117,16 @@ netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar paranamer/2.8//paranamer-2.8.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -ratis-client/2.5.1//ratis-client-2.5.1.jar -ratis-common/2.5.1//ratis-common-2.5.1.jar -ratis-grpc/2.5.1//ratis-grpc-2.5.1.jar -ratis-metrics/2.5.1//ratis-metrics-2.5.1.jar -ratis-netty/2.5.1//ratis-netty-2.5.1.jar -ratis-proto/2.5.1//ratis-proto-2.5.1.jar -ratis-server-api/2.5.1//ratis-server-api-2.5.1.jar -ratis-server/2.5.1//ratis-server-2.5.1.jar -ratis-shell/2.5.1//ratis-shell-2.5.1.jar -ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +ratis-client/3.0.1//ratis-client-3.0.1.jar +ratis-common/3.0.1//ratis-common-3.0.1.jar +ratis-grpc/3.0.1//ratis-grpc-3.0.1.jar +ratis-metrics-default/3.0.1/ratis-metrics-default-3.0.1.jar +ratis-netty/3.0.1//ratis-netty-3.0.1.jar +ratis-proto/3.0.1//ratis-proto-3.0.1.jar +ratis-server-api/3.0.1//ratis-server-api-3.0.1.jar +ratis-server/3.0.1//ratis-server-3.0.1.jar +ratis-shell/3.0.1//ratis-shell-3.0.1.jar +ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar reflections/0.10.2//reflections-0.10.2.jar rocksdbjni/8.11.3//rocksdbjni-8.11.3.jar scala-library/2.12.18//scala-library-2.12.18.jar diff --git a/docs/configuration/ha.md b/docs/configuration/ha.md index 15044ea6138..c88c1d2ad71 100644 --- a/docs/configuration/ha.md +++ b/docs/configuration/ha.md @@ -25,5 +25,6 @@ license: | | celeborn.master.ha.node.<id>.port | 9097 | false | Port to bind of master node in HA mode. | 0.3.0 | celeborn.ha.master.node.<id>.port | | celeborn.master.ha.node.<id>.ratis.port | 9872 | false | Ratis port to bind of master node in HA mode. | 0.3.0 | celeborn.ha.master.node.<id>.ratis.port | | celeborn.master.ha.ratis.raft.rpc.type | netty | false | RPC type for Ratis, available options: netty, grpc. | 0.3.0 | celeborn.ha.master.ratis.raft.rpc.type | -| celeborn.master.ha.ratis.raft.server.storage.dir | /tmp/ratis | false | | 0.3.0 | celeborn.ha.master.ratis.raft.server.storage.dir | +| celeborn.master.ha.ratis.raft.server.storage.dir | /tmp/ratis | false | Root storage directory to hold RaftServer data. | 0.3.0 | celeborn.ha.master.ratis.raft.server.storage.dir | +| celeborn.master.ha.ratis.raft.server.storage.startup.option | RECOVER | false | Startup option of RaftServer storage. Available options: RECOVER, FORMAT. | 0.5.0 | | diff --git a/master/pom.xml b/master/pom.xml index bad0dfabf25..abe0614739f 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -74,6 +74,10 @@ org.apache.ratis ratis-shell + + org.apache.ratis + ratis-metrics-default + org.apache.logging.log4j log4j-slf4j-impl diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java index f0a7560d780..0c41183737a 100644 --- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java +++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HARaftServer.java @@ -50,6 +50,7 @@ import org.apache.ratis.rpc.SupportedRpcType; import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.RaftServerConfigKeys; +import org.apache.ratis.server.storage.RaftStorage.StartupOption; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.util.LifeCycle; import org.apache.ratis.util.SizeInBytes; @@ -144,6 +145,8 @@ private HARaftServer( .setProperties(serverProperties) .setParameters(sslParameters) .setStateMachine(masterStateMachine) + // RATIS-1677. Do not auto format RaftStorage in RECOVER. + .setOption(StartupOption.valueOf(conf.haMasterRatisStorageStartupOption())) .build(); StringBuilder raftPeersStr = new StringBuilder(); @@ -306,8 +309,22 @@ private RaftProperties newRaftProperties(CelebornConf conf, RpcType rpc) { // Set RAFT segment pre-allocated size long raftSegmentPreallocatedSize = conf.haMasterRatisLogPreallocatedSize(); + long raftSegmentWriteBufferSize = conf.haMasterRatisLogWriteBufferSize(); int logAppenderQueueNumElements = conf.haMasterRatisLogAppenderQueueNumElements(); long logAppenderQueueByteLimit = conf.haMasterRatisLogAppenderQueueBytesLimit(); + // RATIS-589. Eliminate buffer copying in SegmentedRaftLogOutputStream. + // 4 bytes (serialized size) + logAppenderQueueByteLimit + 4 bytes (checksum) + if (raftSegmentWriteBufferSize < logAppenderQueueByteLimit + 8) { + throw new IllegalArgumentException( + CelebornConf.HA_MASTER_RATIS_LOG_WRITE_BUFFER_SIZE().key() + + " (= " + + raftSegmentWriteBufferSize + + ") is less than " + + CelebornConf.HA_MASTER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT().key() + + " + 8 (= " + + (logAppenderQueueByteLimit + 8) + + ")"); + } boolean shouldInstallSnapshot = conf.haMasterRatisLogInstallSnapshotEnabled(); RaftServerConfigKeys.Log.Appender.setBufferElementLimit( properties, logAppenderQueueNumElements); @@ -315,6 +332,8 @@ private RaftProperties newRaftProperties(CelebornConf conf, RpcType rpc) { properties, SizeInBytes.valueOf(logAppenderQueueByteLimit)); RaftServerConfigKeys.Log.setPreallocatedSize( properties, SizeInBytes.valueOf(raftSegmentPreallocatedSize)); + RaftServerConfigKeys.Log.setWriteBufferSize( + properties, SizeInBytes.valueOf(raftSegmentWriteBufferSize)); RaftServerConfigKeys.Log.Appender.setInstallSnapshotEnabled(properties, shouldInstallSnapshot); int logPurgeGap = conf.haMasterRatisLogPurgeGap(); RaftServerConfigKeys.Log.setPurgeGap(properties, logPurgeGap); diff --git a/pom.xml b/pom.xml index 307b952274c..0b7fd07af96 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ 4.1.109.Final 1.77 3.21.7 - 2.5.1 + 3.0.1 3.2.16 1.7.36 1.0.6 @@ -283,6 +283,11 @@ + + org.apache.ratis + ratis-metrics-default + ${ratis.version} + org.scala-lang scala-library diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala index c8a61cbaae1..c616237b0e0 100644 --- a/project/CelebornBuild.scala +++ b/project/CelebornBuild.scala @@ -56,7 +56,7 @@ object Dependencies { val metricsVersion = "3.2.6" val mockitoVersion = "4.11.0" val nettyVersion = "4.1.109.Final" - val ratisVersion = "2.5.1" + val ratisVersion = "3.0.1" val roaringBitmapVersion = "1.0.6" val rocksdbJniVersion = "8.11.3" val jacksonVersion = "2.15.3" @@ -127,6 +127,7 @@ object Dependencies { val ratisClient = "org.apache.ratis" % "ratis-client" % ratisVersion val ratisCommon = "org.apache.ratis" % "ratis-common" % ratisVersion val ratisGrpc = "org.apache.ratis" % "ratis-grpc" % ratisVersion + val ratisMetricsDefault = "org.apache.ratis" % "ratis-metrics-default" % ratisVersion val ratisNetty = "org.apache.ratis" % "ratis-netty" % ratisVersion val ratisServer = "org.apache.ratis" % "ratis-server" % ratisVersion val ratisShell = "org.apache.ratis" % "ratis-shell" % ratisVersion excludeAll( @@ -551,6 +552,7 @@ object CelebornMaster { Dependencies.ratisClient, Dependencies.ratisCommon, Dependencies.ratisGrpc, + Dependencies.ratisMetricsDefault, Dependencies.ratisNetty, Dependencies.ratisServer, Dependencies.ratisShell