From 779bca69e287bde1600ef59b11067cb5a7a60f32 Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 13 Aug 2024 12:50:39 +0200 Subject: [PATCH] fix example pipeline --- composable_pipeline_1.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/composable_pipeline_1.py b/composable_pipeline_1.py index 18e8868c0b..a9f1e842d2 100644 --- a/composable_pipeline_1.py +++ b/composable_pipeline_1.py @@ -67,16 +67,16 @@ def o(): print("RUNNING LOCAL SNAPSHOT EXTRACTION") lake_pipeline.run( - duck_pipeline.dataset().customers.iter_df(), - loader_file_format="jsonl", + duck_pipeline.dataset().customers.iter_df(500), + loader_file_format="parquet", table_name="customers", write_disposition="replace", ) lake_pipeline.run( duck_pipeline.dataset().sql( "SELECT * FROM orders WHERE orders.order_day = 'tuesday'" - ).iter_df(), - loader_file_format="jsonl", + ).iter_df(500), + loader_file_format="parquet", table_name="orders", write_disposition="replace", ) @@ -100,8 +100,8 @@ def o(): "SELECT orders.*, customers.name FROM orders LEFT JOIN customers ON" " orders.customer_id = customers.id" ), - ).iter_df(), - loader_file_format="jsonl", + ).iter_df(500), + loader_file_format="parquet", table_name="customers", write_disposition="replace", )