Skip to content

Commit

Permalink
fix: use table config target file size
Browse files Browse the repository at this point in the history
  • Loading branch information
ion-elgreco committed Aug 21, 2024
1 parent 1f45881 commit 1893da3
Showing 1 changed file with 4 additions and 2 deletions.
6 changes: 4 additions & 2 deletions crates/core/src/operations/write.rs
Original file line number Diff line number Diff line change
Expand Up @@ -376,7 +376,7 @@ async fn write_execution_plan_with_predicate(
plan: Arc<dyn ExecutionPlan>,
partition_columns: Vec<String>,
object_store: ObjectStoreRef,
target_file_size: Option<usize>,
mut target_file_size: Option<usize>,
write_batch_size: Option<usize>,
writer_properties: Option<WriterProperties>,
writer_stats_config: WriterStatsConfig,
Expand All @@ -386,6 +386,9 @@ async fn write_execution_plan_with_predicate(
// the schema and batches were prior constructed with this in mind.
let schema: ArrowSchemaRef = plan.schema();
let checker = if let Some(snapshot) = snapshot {
if target_file_size.is_none() {
target_file_size = Some(snapshot.table_config().target_file_size() as usize)
}
DeltaDataChecker::new(snapshot)
} else {
DeltaDataChecker::empty()
Expand All @@ -398,7 +401,6 @@ async fn write_execution_plan_with_predicate(
}
_ => checker,
};

// Write data to disk
let mut tasks = vec![];
for i in 0..plan.properties().output_partitioning().partition_count() {
Expand Down

0 comments on commit 1893da3

Please sign in to comment.