Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Make duplicate check optional for adding parquet files #1034

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
+59 −53
Open

feat: Make duplicate check optional for adding parquet files #1034

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
012f111
feat: Make duplicate check optional for adding parquet files
jonathanc-n Mar 3, 2025
f505488
Merge branch 'main' into check-duplicate
jonathanc-n Mar 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 53 additions & 47 deletions crates/iceberg/src/transaction.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,11 @@ impl<'a> FastAppendAction<'a> {

/// Adds existing parquet files
#[allow(dead_code)]
async fn add_parquet_files(mut self, file_path: Vec<String>) -> Result<Transaction<'a>> {
async fn add_parquet_files(
mut self,
file_path: Vec<String>,
check_duplicate: bool,
) -> Result<Transaction<'a>> {
if !self
.snapshot_produce_action
.tx
Expand All @@ -236,57 +240,59 @@ impl<'a> FastAppendAction<'a> {

self.add_data_files(data_files)?;

self.apply().await
self.apply(check_duplicate).await
}

/// Finished building the action and apply it to the transaction.
pub async fn apply(self) -> Result<Transaction<'a>> {
pub async fn apply(self, check_duplicate: bool) -> Result<Transaction<'a>> {
// Checks duplicate files
let new_files: HashSet<&str> = self
.snapshot_produce_action
.added_data_files
.iter()
.map(|df| df.file_path.as_str())
.collect();

let mut manifest_stream = self
.snapshot_produce_action
.tx
.table
.inspect()
.manifests()
.scan()
.await?;
let mut referenced_files = Vec::new();

while let Some(batch) = manifest_stream.try_next().await? {
let file_path_array = batch
.column(1)
.as_any()
.downcast_ref::<StringArray>()
.ok_or_else(|| {
Error::new(
ErrorKind::DataInvalid,
"Failed to downcast file_path column to StringArray",
)
})?;

for i in 0..batch.num_rows() {
let file_path = file_path_array.value(i);
if new_files.contains(file_path) {
referenced_files.push(file_path.to_string());
if check_duplicate {
let new_files: HashSet<&str> = self
.snapshot_produce_action
.added_data_files
.iter()
.map(|df| df.file_path.as_str())
.collect();

let mut manifest_stream = self
.snapshot_produce_action
.tx
.table
.inspect()
.manifests()
.scan()
.await?;
let mut referenced_files = Vec::new();

while let Some(batch) = manifest_stream.try_next().await? {
let file_path_array = batch
.column(1)
.as_any()
.downcast_ref::<StringArray>()
.ok_or_else(|| {
Error::new(
ErrorKind::DataInvalid,
"Failed to downcast file_path column to StringArray",
)
})?;

for i in 0..batch.num_rows() {
let file_path = file_path_array.value(i);
if new_files.contains(file_path) {
referenced_files.push(file_path.to_string());
}
}
}
}

if !referenced_files.is_empty() {
return Err(Error::new(
ErrorKind::DataInvalid,
format!(
"Cannot add files that are already referenced by table, files: {}",
referenced_files.join(", ")
),
));
if !referenced_files.is_empty() {
return Err(Error::new(
ErrorKind::DataInvalid,
format!(
"Cannot add files that are already referenced by table, files: {}",
referenced_files.join(", ")
),
));
}
}

self.snapshot_produce_action
Expand Down Expand Up @@ -884,7 +890,7 @@ mod tests {
.build()
.unwrap();
action.add_data_files(vec![data_file.clone()]).unwrap();
let tx = action.apply().await.unwrap();
let tx = action.apply(true).await.unwrap();

// check updates and requirements
assert!(
Expand Down Expand Up @@ -971,7 +977,7 @@ mod tests {

// Attempt to add the existing Parquet files with fast append.
let new_tx = fast_append_action
.add_parquet_files(file_paths.clone())
.add_parquet_files(file_paths.clone(), true)
.await
.expect("Adding existing Parquet files should succeed");

Expand Down
4 changes: 2 additions & 2 deletions crates/integration_tests/tests/shared_tests/append_data_file_test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ async fn test_append_data_file() {
let tx = Transaction::new(&table);
let mut append_action = tx.fast_append(None, vec![]).unwrap();
append_action.add_data_files(data_file.clone()).unwrap();
let tx = append_action.apply().await.unwrap();
let tx = append_action.apply(true).await.unwrap();
let table = tx.commit(&rest_catalog).await.unwrap();

// check result
Expand All @@ -134,7 +134,7 @@ async fn test_append_data_file() {
let tx = Transaction::new(&table);
let mut append_action = tx.fast_append(None, vec![]).unwrap();
append_action.add_data_files(data_file.clone()).unwrap();
let tx = append_action.apply().await.unwrap();
let tx = append_action.apply(true).await.unwrap();
let table = tx.commit(&rest_catalog).await.unwrap();

// check result again
Expand Down
2 changes: 1 addition & 1 deletion crates/integration_tests/tests/shared_tests/append_partition_data_file_test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ async fn test_append_partition_data_file() {
append_action
.add_data_files(data_file_valid.clone())
.unwrap();
let tx = append_action.apply().await.unwrap();
let tx = append_action.apply(true).await.unwrap();
let table = tx.commit(&rest_catalog).await.unwrap();

// check result
Expand Down
4 changes: 2 additions & 2 deletions crates/integration_tests/tests/shared_tests/conflict_commit_test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,12 +92,12 @@ async fn test_append_data_file_conflict() {
let tx1 = Transaction::new(&table);
let mut append_action = tx1.fast_append(None, vec![]).unwrap();
append_action.add_data_files(data_file.clone()).unwrap();
let tx1 = append_action.apply().await.unwrap();
let tx1 = append_action.apply(true).await.unwrap();

let tx2 = Transaction::new(&table);
let mut append_action = tx2.fast_append(None, vec![]).unwrap();
append_action.add_data_files(data_file.clone()).unwrap();
let tx2 = append_action.apply().await.unwrap();
let tx2 = append_action.apply(true).await.unwrap();
let table = tx2
.commit(&rest_catalog)
.await
Expand Down
2 changes: 1 addition & 1 deletion crates/integration_tests/tests/shared_tests/scan_all_type.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,7 @@ async fn test_scan_all_type() {
let tx = Transaction::new(&table);
let mut append_action = tx.fast_append(None, vec![]).unwrap();
append_action.add_data_files(data_file.clone()).unwrap();
let tx = append_action.apply().await.unwrap();
let tx = append_action.apply(true).await.unwrap();
let table = tx.commit(&rest_catalog).await.unwrap();

// check result
Expand Down
Loading