init

Signed-off-by: Weichen Xu <[email protected]>
WeichenXu123 · Jul 19, 2024 · c41421c · c41421c
1 parent d3a9055
commit c41421c
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/python/pyspark/ml/util.py b/python/pyspark/ml/util.py
@@ -464,7 +464,8 @@ def saveMetadata(
         metadataJson = DefaultParamsWriter._get_metadata_to_save(
             instance, sc, extraMetadata, paramMap
         )
-        sc.parallelize([metadataJson], 1).saveAsTextFile(metadataPath)
+        spark = SparkSession.getActiveSession()
+        spark.createDataFrame([(metadataJson,)], schema=["value"]).repartition(1).write.text(metadataPath)
 
     @staticmethod
     def _get_metadata_to_save(
@@ -577,7 +578,8 @@ def loadMetadata(path: str, sc: "SparkContext", expectedClassName: str = "") ->
             If non empty, this is checked against the loaded metadata.
         """
         metadataPath = os.path.join(path, "metadata")
-        metadataStr = sc.textFile(metadataPath, 1).first()
+        spark = SparkSession.getActiveSession()
+        metadataStr = spark.read.text(metadataPath).first()[0]
         loadedVals = DefaultParamsReader._parseMetaData(metadataStr, expectedClassName)
         return loadedVals