diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java index b1763634bc065..6a529f010562b 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java @@ -193,7 +193,7 @@ public Option getProperty(EngineProperty key) { @Override public void setJobStatus(String activeModule, String activityDescription) { - javaSparkContext.setJobGroup(activeModule, activityDescription); + javaSparkContext.setJobDescription(String.format("%s:%s", activeModule, activityDescription)); } @Override diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java index e3d163f077f52..22618ab89a1f4 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java @@ -48,4 +48,23 @@ void testAddRemoveCachedDataIds() { assertEquals(Arrays.asList(1, 2, 3), context.removeCachedDataIds(HoodieDataCacheKey.of(basePath, instantTime))); assertTrue(context.getCachedDataIds(HoodieDataCacheKey.of(basePath, instantTime)).isEmpty()); } + + @Test + void testSetJobStatus() { + // Test data + String jobGroupId = "jobGroupId"; + String initialJobDescription = "initialJobDescription"; + String activeModule = "TestModule"; + String activityDescription = "Running test operation"; + String expectedJobDescription = String.format("%s:%s", activeModule, activityDescription); + context.getJavaSparkContext().setJobGroup(jobGroupId, initialJobDescription); + assertEquals(jobGroupId, context.getJavaSparkContext().getLocalProperty("spark.jobGroup.id")); + assertEquals(initialJobDescription, context.getJavaSparkContext().getLocalProperty("spark.job.description")); + + // Set the job status + context.setJobStatus(activeModule, activityDescription); + assertEquals(expectedJobDescription, context.getJavaSparkContext().getLocalProperty("spark.job.description")); + // Assert jobGroupId does not change + assertEquals(jobGroupId, context.getJavaSparkContext().getLocalProperty("spark.jobGroup.id")); + } }