init

logicalclocks · Dec 9, 2024 · 840824e · 840824e
1 parent ad441ee
commit 840824e
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/utils/python/hsfs_utils.py b/utils/python/hsfs_utils.py
@@ -303,6 +303,9 @@ def offline_fg_materialization(spark: SparkSession, job_conf: Dict[Any, Any], in
         .load()
     )
 
+    # Cache the DataFrame in memory
+    df.cache()
+
     # filter only the necassary entries
     filtered_df = df.filter(expr("CAST(filter(headers, header -> header.key = 'featureGroupId')[0].value AS STRING)") == str(entity._id))
     filtered_df = filtered_df.filter(expr("CAST(filter(headers, header -> header.key = 'subjectId')[0].value AS STRING)") == str(entity.subject["id"]))