[FSTORE-1627] Don't fail Delta delta-streamer on data loss (#420)

logicalclocks · Dec 2, 2024 · d368e5c · d368e5c
1 parent db70457
commit d368e5c
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/utils/python/hsfs_utils.py b/utils/python/hsfs_utils.py
@@ -299,6 +299,7 @@ def offline_fg_materialization(spark: SparkSession, job_conf: Dict[Any, Any], in
         .option("subscribe", entity._online_topic_name)
         .option("startingOffsets", offset_string)
         .option("includeHeaders", "true")
+        .option("failOnDataLoss", "false")
         .load()
         .limit(5000000)
     )