From 0802f6c82d9f623f75f81389717f9762747a3d90 Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Wed, 9 Aug 2023 14:09:31 +0800 Subject: [PATCH 01/12] [CELEBORN-883][WORKER] Optimized configuration checks during MemoryManager initialization --- .../apache/celeborn/common/CelebornConf.scala | 3 +- docs/configuration/worker.md | 2 +- .../deploy/worker/memory/MemoryManager.java | 6 +++- .../worker/memory/MemoryManagerSuiteJ.java | 30 +++++++++++++++++++ 4 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java diff --git a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala index afa8276b0a2..258356757b5 100644 --- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala +++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala @@ -2450,7 +2450,8 @@ object CelebornConf extends Logging { val WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE: ConfigEntry[Double] = buildConf("celeborn.worker.directMemoryRatioToPauseReplicate") .categories("worker") - .doc("If direct memory usage reaches this limit, the worker will stop to receive replication data from other workers.") + .doc("If direct memory usage reaches this limit, the worker will stop to receive replication data from other workers." + + s"This value should be higher than ${WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE.key}.") .version("0.2.0") .doubleConf .createWithDefault(0.95) diff --git a/docs/configuration/worker.md b/docs/configuration/worker.md index d5759148576..36a09163e71 100644 --- a/docs/configuration/worker.md +++ b/docs/configuration/worker.md @@ -38,7 +38,7 @@ license: | | celeborn.worker.directMemoryRatioForMemoryShuffleStorage | 0.0 | Max ratio of direct memory to store shuffle data | 0.2.0 | | celeborn.worker.directMemoryRatioForReadBuffer | 0.1 | Max ratio of direct memory for read buffer | 0.2.0 | | celeborn.worker.directMemoryRatioToPauseReceive | 0.85 | If direct memory usage reaches this limit, the worker will stop to receive data from Celeborn shuffle clients. | 0.2.0 | -| celeborn.worker.directMemoryRatioToPauseReplicate | 0.95 | If direct memory usage reaches this limit, the worker will stop to receive replication data from other workers. | 0.2.0 | +| celeborn.worker.directMemoryRatioToPauseReplicate | 0.95 | If direct memory usage reaches this limit, the worker will stop to receive replication data from other workers. This value should be higher than `celeborn.worker.directMemoryRatioToPauseReceive`.| 0.2.0 | | celeborn.worker.directMemoryRatioToResume | 0.5 | If direct memory usage is less than this limit, worker will resume. | 0.2.0 | | celeborn.worker.fetch.heartbeat.enabled | false | enable the heartbeat from worker to client when fetching data | 0.3.0 | | celeborn.worker.fetch.io.threads | <undefined> | Netty IO thread number of worker to handle client fetch data. The default threads number is the number of flush thread. | 0.2.0 | diff --git a/worker/src/main/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManager.java b/worker/src/main/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManager.java index c0524fa3a6a..7256e83b6b8 100644 --- a/worker/src/main/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManager.java +++ b/worker/src/main/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManager.java @@ -120,7 +120,11 @@ private MemoryManager(CelebornConf conf) { .invoke(); Preconditions.checkArgument(maxDirectorMemory > 0); - Preconditions.checkArgument(pauseReplicateRatio > pausePushDataRatio); + Preconditions.checkArgument( + pauseReplicateRatio > pausePushDataRatio, + "Invalid config, {} should be greater than {}", + CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE().key(), + CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE().key()); Preconditions.checkArgument(pausePushDataRatio > resumeRatio); Preconditions.checkArgument(resumeRatio > (readBufferRatio + shuffleStorageRatio)); diff --git a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java new file mode 100644 index 00000000000..b6bbaad7b29 --- /dev/null +++ b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java @@ -0,0 +1,30 @@ +package org.apache.celeborn.service.deploy.worker.memory; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.celeborn.common.CelebornConf; + +public class MemoryManagerSuiteJ { + + @Test + public void testInitMemoryManagerWithInvalidConfig() { + CelebornConf conf = new CelebornConf(); + conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE(), "0.9"); + conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE(), "0.95"); + try { + MemoryManager.initialize(conf); + // should throw exception before here + Assert.fail("MemoryManager initialize should throw exception with invalid configuration"); + } catch (IllegalArgumentException iae) { + Assert.assertEquals( + iae.getMessage(), + String.format( + "Invalid config, " + "{} should be greater than {}", + CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE().key(), + CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE().key())); + } catch (Exception e) { + Assert.fail("With unexpected exception" + e); + } + } +} From a2390b0f71faf571f3c29fd998e92053ff8f5cf8 Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Wed, 9 Aug 2023 14:15:50 +0800 Subject: [PATCH 02/12] Add license header --- .../worker/memory/MemoryManagerSuiteJ.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java index b6bbaad7b29..4030bad9cc5 100644 --- a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java +++ b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.celeborn.service.deploy.worker.memory; import org.junit.Assert; From 877f5dd2420ed2bb4c377cf8491cde35b7b5a9aa Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Wed, 9 Aug 2023 14:35:44 +0800 Subject: [PATCH 03/12] Fix --- docs/configuration/worker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/worker.md b/docs/configuration/worker.md index 36a09163e71..eadd04dbe65 100644 --- a/docs/configuration/worker.md +++ b/docs/configuration/worker.md @@ -38,7 +38,7 @@ license: | | celeborn.worker.directMemoryRatioForMemoryShuffleStorage | 0.0 | Max ratio of direct memory to store shuffle data | 0.2.0 | | celeborn.worker.directMemoryRatioForReadBuffer | 0.1 | Max ratio of direct memory for read buffer | 0.2.0 | | celeborn.worker.directMemoryRatioToPauseReceive | 0.85 | If direct memory usage reaches this limit, the worker will stop to receive data from Celeborn shuffle clients. | 0.2.0 | -| celeborn.worker.directMemoryRatioToPauseReplicate | 0.95 | If direct memory usage reaches this limit, the worker will stop to receive replication data from other workers. This value should be higher than `celeborn.worker.directMemoryRatioToPauseReceive`.| 0.2.0 | +| celeborn.worker.directMemoryRatioToPauseReplicate | 0.95 | If direct memory usage reaches this limit, the worker will stop to receive replication data from other workers.This value should be higher than celeborn.worker.directMemoryRatioToPauseReceive. | 0.2.0 | | celeborn.worker.directMemoryRatioToResume | 0.5 | If direct memory usage is less than this limit, worker will resume. | 0.2.0 | | celeborn.worker.fetch.heartbeat.enabled | false | enable the heartbeat from worker to client when fetching data | 0.3.0 | | celeborn.worker.fetch.io.threads | <undefined> | Netty IO thread number of worker to handle client fetch data. The default threads number is the number of flush thread. | 0.2.0 | From 9d0d8fceb685e797f8fc8fad48168edf87a338d4 Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Wed, 9 Aug 2023 14:59:06 +0800 Subject: [PATCH 04/12] fix unit test --- .../service/deploy/worker/memory/MemoryManagerSuiteJ.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java index 4030bad9cc5..39d934b655b 100644 --- a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java +++ b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java @@ -27,8 +27,8 @@ public class MemoryManagerSuiteJ { @Test public void testInitMemoryManagerWithInvalidConfig() { CelebornConf conf = new CelebornConf(); - conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE(), "0.9"); - conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE(), "0.95"); + conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE(), "0.95"); + conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE(), "0.85"); try { MemoryManager.initialize(conf); // should throw exception before here From a10b318924135e3dcae7fd42c6fc00f6d10d5164 Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Wed, 9 Aug 2023 15:13:56 +0800 Subject: [PATCH 05/12] fix unit test --- .../service/deploy/worker/memory/MemoryManager.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/worker/src/main/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManager.java b/worker/src/main/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManager.java index 7256e83b6b8..0e52a5c085d 100644 --- a/worker/src/main/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManager.java +++ b/worker/src/main/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManager.java @@ -122,9 +122,10 @@ private MemoryManager(CelebornConf conf) { Preconditions.checkArgument(maxDirectorMemory > 0); Preconditions.checkArgument( pauseReplicateRatio > pausePushDataRatio, - "Invalid config, {} should be greater than {}", - CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE().key(), - CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE().key()); + String.format( + "Invalid config, {} should be greater than {}", + CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE().key(), + CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE().key())); Preconditions.checkArgument(pausePushDataRatio > resumeRatio); Preconditions.checkArgument(resumeRatio > (readBufferRatio + shuffleStorageRatio)); From 26075c5eb8f1322ecc0f255c9cf9163e7f6a6193 Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Wed, 9 Aug 2023 15:36:57 +0800 Subject: [PATCH 06/12] fix unit test --- .../service/deploy/worker/memory/MemoryManagerSuiteJ.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java index 39d934b655b..1cb6513a6a0 100644 --- a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java +++ b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java @@ -35,11 +35,11 @@ public void testInitMemoryManagerWithInvalidConfig() { Assert.fail("MemoryManager initialize should throw exception with invalid configuration"); } catch (IllegalArgumentException iae) { Assert.assertEquals( - iae.getMessage(), String.format( - "Invalid config, " + "{} should be greater than {}", + "Invalid config, {} should be greater than {}", CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE().key(), - CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE().key())); + CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE().key()), + iae.getMessage()); } catch (Exception e) { Assert.fail("With unexpected exception" + e); } From 05e5a760ea95daf63e2cfaa99102395d01446511 Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Thu, 10 Aug 2023 10:24:24 +0800 Subject: [PATCH 07/12] Rest MemoryManager Before test --- .../service/deploy/worker/memory/MemoryManagerSuiteJ.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java index 1cb6513a6a0..e637ab33a61 100644 --- a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java +++ b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java @@ -18,12 +18,18 @@ package org.apache.celeborn.service.deploy.worker.memory; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.apache.celeborn.common.CelebornConf; public class MemoryManagerSuiteJ { + @Before + public void resetMemoryManager() { + MemoryManager.reset(); + } + @Test public void testInitMemoryManagerWithInvalidConfig() { CelebornConf conf = new CelebornConf(); From ad099d99f92dfa1cf788449c381c443c363709fa Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Thu, 10 Aug 2023 12:02:38 +0800 Subject: [PATCH 08/12] fix comments --- .../apache/celeborn/common/CelebornConf.scala | 2 +- .../deploy/worker/memory/MemoryManager.java | 6 ++-- .../deploy/memory/MemoryManagerSuite.scala | 28 +++++++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 worker/src/test/scala/org/apache/celeborn/service/deploy/memory/MemoryManagerSuite.scala diff --git a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala index 258356757b5..fdc4988b104 100644 --- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala +++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala @@ -2450,7 +2450,7 @@ object CelebornConf extends Logging { val WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE: ConfigEntry[Double] = buildConf("celeborn.worker.directMemoryRatioToPauseReplicate") .categories("worker") - .doc("If direct memory usage reaches this limit, the worker will stop to receive replication data from other workers." + + .doc("If direct memory usage reaches this limit, the worker will stop to receive replication data from other workers. " + s"This value should be higher than ${WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE.key}.") .version("0.2.0") .doubleConf diff --git a/worker/src/main/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManager.java b/worker/src/main/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManager.java index 0e52a5c085d..758ddbb8ca6 100644 --- a/worker/src/main/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManager.java +++ b/worker/src/main/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManager.java @@ -123,9 +123,11 @@ private MemoryManager(CelebornConf conf) { Preconditions.checkArgument( pauseReplicateRatio > pausePushDataRatio, String.format( - "Invalid config, {} should be greater than {}", + "Invalid config, %s(%s) should be greater than %s(%s)", CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE().key(), - CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE().key())); + pauseReplicateRatio, + CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE().key(), + pausePushDataRatio)); Preconditions.checkArgument(pausePushDataRatio > resumeRatio); Preconditions.checkArgument(resumeRatio > (readBufferRatio + shuffleStorageRatio)); diff --git a/worker/src/test/scala/org/apache/celeborn/service/deploy/memory/MemoryManagerSuite.scala b/worker/src/test/scala/org/apache/celeborn/service/deploy/memory/MemoryManagerSuite.scala new file mode 100644 index 00000000000..e3e69c3416c --- /dev/null +++ b/worker/src/test/scala/org/apache/celeborn/service/deploy/memory/MemoryManagerSuite.scala @@ -0,0 +1,28 @@ +package org.apache.celeborn.service.deploy.memory + +import org.scalatest.BeforeAndAfterEach +import org.scalatest.funsuite.AnyFunSuite + +import org.apache.celeborn.common.CelebornConf +import org.apache.celeborn.common.CelebornConf.{WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE, WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE} +import org.apache.celeborn.service.deploy.worker.memory.MemoryManager +class MemoryManagerSuite extends AnyFunSuite with BeforeAndAfterEach { + + // reset the memory manager before each test + override protected def beforeEach(): Unit = { + super.beforeEach() + MemoryManager.reset() + } + + test("Init MemoryManager with invalid configuration") { + val conf = new CelebornConf().set(WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE, 0.95) + .set(WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE, 0.85) + val caught = + intercept[IllegalArgumentException] { + MemoryManager.initialize(conf); + } + assert( + caught.getMessage == s"Invalid config, ${WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE.key}(${0.85}) " + + s"should be greater than ${WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE.key}(${0.95})") + } +} From 801c18ac821eec6c51df5c2fffd66fd7a846ba48 Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Thu, 10 Aug 2023 14:10:12 +0800 Subject: [PATCH 09/12] fix config doc --- docs/configuration/worker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/worker.md b/docs/configuration/worker.md index eadd04dbe65..e724fa2aad7 100644 --- a/docs/configuration/worker.md +++ b/docs/configuration/worker.md @@ -38,7 +38,7 @@ license: | | celeborn.worker.directMemoryRatioForMemoryShuffleStorage | 0.0 | Max ratio of direct memory to store shuffle data | 0.2.0 | | celeborn.worker.directMemoryRatioForReadBuffer | 0.1 | Max ratio of direct memory for read buffer | 0.2.0 | | celeborn.worker.directMemoryRatioToPauseReceive | 0.85 | If direct memory usage reaches this limit, the worker will stop to receive data from Celeborn shuffle clients. | 0.2.0 | -| celeborn.worker.directMemoryRatioToPauseReplicate | 0.95 | If direct memory usage reaches this limit, the worker will stop to receive replication data from other workers.This value should be higher than celeborn.worker.directMemoryRatioToPauseReceive. | 0.2.0 | +| celeborn.worker.directMemoryRatioToPauseReplicate | 0.95 | If direct memory usage reaches this limit, the worker will stop to receive replication data from other workers. This value should be higher than celeborn.worker.directMemoryRatioToPauseReceive. | 0.2.0 | | celeborn.worker.directMemoryRatioToResume | 0.5 | If direct memory usage is less than this limit, worker will resume. | 0.2.0 | | celeborn.worker.fetch.heartbeat.enabled | false | enable the heartbeat from worker to client when fetching data | 0.3.0 | | celeborn.worker.fetch.io.threads | <undefined> | Netty IO thread number of worker to handle client fetch data. The default threads number is the number of flush thread. | 0.2.0 | From 44e0f85c371de6995463d7ccc257d0af2f402bb1 Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Thu, 10 Aug 2023 14:14:25 +0800 Subject: [PATCH 10/12] Add license header --- .../deploy/memory/MemoryManagerSuite.scala | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/worker/src/test/scala/org/apache/celeborn/service/deploy/memory/MemoryManagerSuite.scala b/worker/src/test/scala/org/apache/celeborn/service/deploy/memory/MemoryManagerSuite.scala index e3e69c3416c..2fe410e93b2 100644 --- a/worker/src/test/scala/org/apache/celeborn/service/deploy/memory/MemoryManagerSuite.scala +++ b/worker/src/test/scala/org/apache/celeborn/service/deploy/memory/MemoryManagerSuite.scala @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.celeborn.service.deploy.memory import org.scalatest.BeforeAndAfterEach From 9b1a2cfcdfa3c84e60e1e2d022ab3c8208d56677 Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Thu, 10 Aug 2023 15:09:01 +0800 Subject: [PATCH 11/12] fix unit test --- .../celeborn/service/deploy/memory/MemoryManagerSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/worker/src/test/scala/org/apache/celeborn/service/deploy/memory/MemoryManagerSuite.scala b/worker/src/test/scala/org/apache/celeborn/service/deploy/memory/MemoryManagerSuite.scala index 2fe410e93b2..0e0d627bf15 100644 --- a/worker/src/test/scala/org/apache/celeborn/service/deploy/memory/MemoryManagerSuite.scala +++ b/worker/src/test/scala/org/apache/celeborn/service/deploy/memory/MemoryManagerSuite.scala @@ -39,7 +39,7 @@ class MemoryManagerSuite extends AnyFunSuite with BeforeAndAfterEach { MemoryManager.initialize(conf); } assert( - caught.getMessage == s"Invalid config, ${WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE.key}(${0.85}) " + - s"should be greater than ${WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE.key}(${0.95})") + caught.getMessage == s"Invalid config, ${WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE.key}(0.85) " + + s"should be greater than ${WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE.key}(0.95)") } } From d857bb867b452034ce53b3c3f43b4aa96e7625ad Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Thu, 10 Aug 2023 15:29:59 +0800 Subject: [PATCH 12/12] Remove outdate java unit test --- .../worker/memory/MemoryManagerSuiteJ.java | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java diff --git a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java deleted file mode 100644 index e637ab33a61..00000000000 --- a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/memory/MemoryManagerSuiteJ.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.celeborn.service.deploy.worker.memory; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import org.apache.celeborn.common.CelebornConf; - -public class MemoryManagerSuiteJ { - - @Before - public void resetMemoryManager() { - MemoryManager.reset(); - } - - @Test - public void testInitMemoryManagerWithInvalidConfig() { - CelebornConf conf = new CelebornConf(); - conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE(), "0.95"); - conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE(), "0.85"); - try { - MemoryManager.initialize(conf); - // should throw exception before here - Assert.fail("MemoryManager initialize should throw exception with invalid configuration"); - } catch (IllegalArgumentException iae) { - Assert.assertEquals( - String.format( - "Invalid config, {} should be greater than {}", - CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE().key(), - CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE().key()), - iae.getMessage()); - } catch (Exception e) { - Assert.fail("With unexpected exception" + e); - } - } -}