From 9a38bd9fd6b3d9f79072f8e7b384a55c892f2bd8 Mon Sep 17 00:00:00 2001 From: Jan Kaul Date: Wed, 23 Oct 2024 16:13:04 +0200 Subject: [PATCH] fix partition values --- Cargo.lock | 16 +-- datafusion_iceberg/src/table.rs | 68 +--------- iceberg-rust-spec/src/spec/manifest_list.rs | 48 ++++--- .../src/table/transaction/operation.rs | 62 +++++---- ...-a796-4f56-97b0-a638a6f6d6f4-00001.parquet | Bin 1501 -> 0 bytes ...-5d0b-4c42-9782-f08ec510637e-00001.parquet | Bin 1500 -> 0 bytes ...-21d8-4302-ad47-85c00b092e8b-00001.parquet | Bin 1499 -> 0 bytes ...-1e3c-4905-a73c-5e0ea8ed268f-00001.parquet | Bin 1501 -> 0 bytes ...-1326-447a-97fd-f4827081b3c5.metadata.json | 1 - ...-fcab-4be8-903c-25bc0974156b.metadata.json | 1 - ...-c427-4232-ab45-704a663ff72b.metadata.json | 1 - ...-530f-4120-9563-ccad06de4098.metadata.json | 1 - ...-0663-4f3c-9523-e338aebb25ef.metadata.json | 1 - ...-8882-44ca-9b1d-92c7297d3f2e.metadata.json | 1 - ...-44bb-449f-a942-d936fe75932e.metadata.json | 1 - ...-c882-4021-b385-9d74cb3ecb3b.metadata.json | 1 - ...-3c04-49a0-af8a-a70c5d3fcc16.metadata.json | 1 - ...-c07f-4405-90dc-5bd2a83501f0.metadata.json | 1 - ...-df97-4774-ac2e-9d11002033be.metadata.json | 1 - ...-720c-458b-8cf4-7411ce9f5eea.metadata.json | 1 - ...-c5e9-4483-8a97-bedce302e5d0.metadata.json | 1 - ...-6a8a-4bb7-9d18-e9ea5bb998d5.metadata.json | 1 - ...6760f0-4f6c-4b23-b907-0a5a174e3863-m0.avro | Bin 6447 -> 0 bytes ...-a02b-11e9-ae9c-1bb7bc9eca94.metadata.json | 128 ------------------ ...-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro | Bin 3766 -> 0 bytes 25 files changed, 69 insertions(+), 267 deletions(-) delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/data/vendor_id=1/00000-0-03c9b632-a796-4f56-97b0-a638a6f6d6f4-00001.parquet delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/data/vendor_id=1/00003-3-ae86257a-5d0b-4c42-9782-f08ec510637e-00001.parquet delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/data/vendor_id=2/00001-1-2a1bfa65-21d8-4302-ad47-85c00b092e8b-00001.parquet delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/data/vendor_id=2/00002-2-22cded08-1e3c-4905-a73c-5e0ea8ed268f-00001.parquet delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-0b944f80-1326-447a-97fd-f4827081b3c5.metadata.json delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-1a907b10-fcab-4be8-903c-25bc0974156b.metadata.json delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-1c785339-c427-4232-ab45-704a663ff72b.metadata.json delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-2b717789-530f-4120-9563-ccad06de4098.metadata.json delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-300c39ad-0663-4f3c-9523-e338aebb25ef.metadata.json delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-5b83190d-8882-44ca-9b1d-92c7297d3f2e.metadata.json delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-730360ca-44bb-449f-a942-d936fe75932e.metadata.json delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-9b3fb15d-c882-4021-b385-9d74cb3ecb3b.metadata.json delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-a5662485-3c04-49a0-af8a-a70c5d3fcc16.metadata.json delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-bd4a6f4b-c07f-4405-90dc-5bd2a83501f0.metadata.json delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-cd9e2028-df97-4774-ac2e-9d11002033be.metadata.json delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-e9cb38f8-720c-458b-8cf4-7411ce9f5eea.metadata.json delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-f44e4bfe-c5e9-4483-8a97-bedce302e5d0.metadata.json delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-fb3797e1-6a8a-4bb7-9d18-e9ea5bb998d5.metadata.json delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/7e6760f0-4f6c-4b23-b907-0a5a174e3863-m0.avro delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/fb072c92-a02b-11e9-ae9c-1bb7bc9eca94.metadata.json delete mode 100644 iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro diff --git a/Cargo.lock b/Cargo.lock index 79536b0a..d30cb145 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1752,7 +1752,7 @@ checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" dependencies = [ "futures-core", "futures-sink", - "spin 0.9.8", + "spin", ] [[package]] @@ -2649,11 +2649,11 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.5.2", + "spin", ] [[package]] @@ -3827,7 +3827,7 @@ dependencies = [ "cfg-if", "getrandom", "libc", - "spin 0.9.8", + "spin", "untrusted", "windows-sys 0.52.0", ] @@ -4349,12 +4349,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" diff --git a/datafusion_iceberg/src/table.rs b/datafusion_iceberg/src/table.rs index e2c088c2..17cd6d6e 100644 --- a/datafusion_iceberg/src/table.rs +++ b/datafusion_iceberg/src/table.rs @@ -596,20 +596,14 @@ impl DataSink for IcebergDataSink { #[cfg(test)] mod tests { - use datafusion::{ - arrow::{ - array::{Float32Array, Int64Array}, - record_batch::RecordBatch, - }, - prelude::SessionContext, - }; + use datafusion::{arrow::array::Int64Array, prelude::SessionContext}; use iceberg_rust::spec::{ partition::{PartitionField, Transform}, schema::Schema, types::{PrimitiveType, StructField, StructType, Type}, }; use iceberg_rust::{ - catalog::{identifier::Identifier, tabular::Tabular, Catalog}, + catalog::Catalog, spec::{ partition::PartitionSpec, view_metadata::{Version, ViewRepresentation}, @@ -618,64 +612,10 @@ mod tests { view::View, }; use iceberg_sql_catalog::SqlCatalog; - use object_store::{local::LocalFileSystem, memory::InMemory, ObjectStore}; + use object_store::{memory::InMemory, ObjectStore}; use std::sync::Arc; - use crate::{catalog::catalog::IcebergCatalog, error::Error, DataFusionTable}; - - #[tokio::test] - pub async fn test_datafusion_table_scan() { - let object_store: Arc = - Arc::new(LocalFileSystem::new_with_prefix("../iceberg-tests/nyc_taxis").unwrap()); - - let catalog: Arc = Arc::new( - SqlCatalog::new("sqlite://", "test", object_store.clone()) - .await - .unwrap(), - ); - let identifier = Identifier::parse("test.table1", None).unwrap(); - - catalog.clone().register_table(identifier.clone(), "/home/iceberg/warehouse/nyc/taxis/metadata/fb072c92-a02b-11e9-ae9c-1bb7bc9eca94.metadata.json").await.expect("Failed to register table."); - - let table = if let Tabular::Table(table) = catalog - .load_tabular(&identifier) - .await - .expect("Failed to load table") - { - Ok(Arc::new(DataFusionTable::from(table))) - } else { - Err(Error::InvalidFormat( - "Entity returned from catalog".to_string(), - )) - } - .unwrap(); - - let ctx = SessionContext::new(); - - ctx.register_table("nyc_taxis", table).unwrap(); - - let df = ctx - .sql("SELECT vendor_id, MIN(trip_distance) FROM nyc_taxis GROUP BY vendor_id") - .await - .unwrap(); - - // execute the plan - let results: Vec = df.collect().await.expect("Failed to execute query plan."); - - let batch = results - .into_iter() - .find(|batch| batch.num_rows() > 0) - .expect("All record batches are empty"); - - let values = batch - .column(1) - .as_any() - .downcast_ref::() - .expect("Failed to get values from batch."); - - // Value can either be 0.9 or 1.8 - assert!(((1.35 - values.value(0)).abs() - 0.45).abs() < 0.001) - } + use crate::{catalog::catalog::IcebergCatalog, DataFusionTable}; #[tokio::test] pub async fn test_datafusion_table_insert() { diff --git a/iceberg-rust-spec/src/spec/manifest_list.rs b/iceberg-rust-spec/src/spec/manifest_list.rs index ef828a0c..cf1a8d64 100644 --- a/iceberg-rust-spec/src/spec/manifest_list.rs +++ b/iceberg-rust-spec/src/spec/manifest_list.rs @@ -5,6 +5,7 @@ use std::{ io::Read, iter::{repeat, Map, Repeat, Zip}, + sync::OnceLock, }; use apache_avro::{types::Value as AvroValue, Reader as AvroReader, Schema as AvroSchema}; @@ -42,12 +43,13 @@ impl<'a, 'metadata, R: Read> Iterator for ManifestListReader<'a, 'metadata, R> { impl<'a, 'metadata, R: Read> ManifestListReader<'a, 'metadata, R> { /// Create a new ManifestFile reader - pub fn new( - reader: R, - table_metadata: &'metadata TableMetadata, - ) -> Result { + pub fn new(reader: R, table_metadata: &'metadata TableMetadata) -> Result { + let schema: &AvroSchema = match table_metadata.format_version { + FormatVersion::V1 => manifest_list_schema_v1(), + FormatVersion::V2 => manifest_list_schema_v2(), + }; Ok(Self { - reader: AvroReader::new(reader)? + reader: AvroReader::with_schema(&schema, reader)? .zip(repeat(table_metadata)) .map(avro_value_to_manifest_file), }) @@ -404,11 +406,11 @@ impl FieldSummary { } } -impl ManifestListEntry { - /// Get schema of the manifest list - pub fn schema(format_version: &FormatVersion) -> Result { - let schema = match format_version { - FormatVersion::V1 => r#" +pub fn manifest_list_schema_v1() -> &'static AvroSchema { + static MANIFEST_LIST_SCHEMA_V1: OnceLock = OnceLock::new(); + MANIFEST_LIST_SCHEMA_V1.get_or_init(|| { + AvroSchema::parse_str( + r#" { "type": "record", "name": "manifest_file", @@ -544,9 +546,16 @@ impl ManifestListEntry { } ] } - "# - .to_owned(), - &FormatVersion::V2 => r#" + "#, + ) + .unwrap() + }) +} +pub fn manifest_list_schema_v2() -> &'static AvroSchema { + static MANIFEST_LIST_SCHEMA_V2: OnceLock = OnceLock::new(); + MANIFEST_LIST_SCHEMA_V2.get_or_init(|| { + AvroSchema::parse_str( + r#" { "type": "record", "name": "manifest_file", @@ -673,11 +682,10 @@ impl ManifestListEntry { } ] } - "# - .to_owned(), - }; - AvroSchema::parse_str(&schema).map_err(Into::into) - } + "#, + ) + .unwrap() + }) } /// Convert an avro value to a [ManifestFile] according to the provided format version @@ -771,7 +779,7 @@ mod tests { key_metadata: None, }; - let schema = ManifestListEntry::schema(&FormatVersion::V2).unwrap(); + let schema = manifest_list_schema_v2(); let mut writer = apache_avro::Writer::new(&schema, Vec::new()); @@ -851,7 +859,7 @@ mod tests { key_metadata: None, }; - let schema = ManifestListEntry::schema(&FormatVersion::V1).unwrap(); + let schema = manifest_list_schema_v1(); let mut writer = apache_avro::Writer::new(&schema, Vec::new()); diff --git a/iceberg-rust/src/table/transaction/operation.rs b/iceberg-rust/src/table/transaction/operation.rs index 2a2e9f36..18cde6fd 100644 --- a/iceberg-rust/src/table/transaction/operation.rs +++ b/iceberg-rust/src/table/transaction/operation.rs @@ -7,20 +7,23 @@ use std::{ sync::Arc, }; -use apache_avro::from_value; use futures::{lock::Mutex, stream, StreamExt, TryStreamExt}; -use iceberg_rust_spec::spec::{ - manifest::{partition_value_schema, DataFile, ManifestEntry, ManifestWriter, Status}, - manifest_list::{Content, FieldSummary, ManifestListEntry, ManifestListEntryEnum}, - partition::PartitionField, - schema::Schema, - snapshot::{ - generate_snapshot_id, SnapshotBuilder, SnapshotReference, SnapshotRetention, Summary, +use iceberg_rust_spec::{error::Error as SpecError, spec::table_metadata::TableMetadata}; +use iceberg_rust_spec::{manifest_list::ManifestListReader, util::strip_prefix}; +use iceberg_rust_spec::{ + manifest_list::{manifest_list_schema_v1, manifest_list_schema_v2}, + spec::{ + manifest::{partition_value_schema, DataFile, ManifestEntry, ManifestWriter, Status}, + manifest_list::{Content, FieldSummary, ManifestListEntry}, + partition::PartitionField, + schema::Schema, + snapshot::{ + generate_snapshot_id, SnapshotBuilder, SnapshotReference, SnapshotRetention, Summary, + }, + values::{Struct, Value}, }, - values::{Struct, Value}, + table_metadata::FormatVersion, }; -use iceberg_rust_spec::util::strip_prefix; -use iceberg_rust_spec::{error::Error as SpecError, spec::table_metadata::TableMetadata}; use object_store::ObjectStore; use crate::{ @@ -102,8 +105,10 @@ impl Operation { }, )?); - let manifest_list_schema = - ManifestListEntry::schema(&table_metadata.format_version)?; + let manifest_list_schema = match table_metadata.format_version { + FormatVersion::V1 => manifest_list_schema_v1(), + FormatVersion::V2 => manifest_list_schema_v2(), + }; let manifest_list_writer = Arc::new(Mutex::new(apache_avro::Writer::new( &manifest_list_schema, @@ -128,23 +133,14 @@ impl Operation { let existing_manifest_iter = if let Some(manifest_list_bytes) = &manifest_list_bytes { let manifest_list_reader = - apache_avro::Reader::new(manifest_list_bytes.as_ref())?; + ManifestListReader::new(manifest_list_bytes.as_ref(), &table_metadata)?; Some(stream::iter(manifest_list_reader).filter_map(|manifest| { let datafiles = datafiles.clone(); let existing_partitions = existing_partitions.clone(); let partition_spec = partition_spec.clone(); async move { - let manifest = manifest - .map_err(Into::into) - .and_then(|value| { - ManifestListEntry::try_from_enum( - from_value::(&value)?, - table_metadata, - ) - }) - .unwrap(); - + let manifest = manifest.ok()?; if let Some(summary) = &manifest.partitions { let partition_values = partition_values_in_bounds( summary, @@ -384,8 +380,10 @@ impl Operation { (ManifestStatus::New(manifest), vec![partition_value.clone()]) }); - let manifest_list_schema = - ManifestListEntry::schema(&table_metadata.format_version)?; + let manifest_list_schema = match table_metadata.format_version { + FormatVersion::V1 => manifest_list_schema_v1(), + FormatVersion::V2 => manifest_list_schema_v2(), + }; let manifest_list_writer = Arc::new(Mutex::new(apache_avro::Writer::new( &manifest_list_schema, @@ -624,10 +622,12 @@ fn update_partitions( partition_values: &Struct, partition_columns: &[PartitionField], ) -> Result<(), Error> { - for (field, summary) in partition_columns.iter().zip(partitions.iter_mut()) { - let value = &partition_values.get(field.name()).and_then(|x| x.as_ref()); + for (field, summary) in partition_columns.into_iter().zip(partitions.iter_mut()) { + let value = partition_values.get(field.name()).and_then(|x| x.as_ref()); if let Some(value) = value { - if let Some(lower_bound) = &mut summary.lower_bound { + if summary.lower_bound.is_none() { + summary.lower_bound = Some(value.clone()); + } else if let Some(lower_bound) = &mut summary.lower_bound { match (value, lower_bound) { (Value::Int(val), Value::Int(current)) => { if *current > *val { @@ -672,7 +672,9 @@ fn update_partitions( _ => {} } } - if let Some(upper_bound) = &mut summary.upper_bound { + if summary.upper_bound.is_none() { + summary.upper_bound = Some(value.clone()); + } else if let Some(upper_bound) = &mut summary.upper_bound { match (value, upper_bound) { (Value::Int(val), Value::Int(current)) => { if *current < *val { diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/data/vendor_id=1/00000-0-03c9b632-a796-4f56-97b0-a638a6f6d6f4-00001.parquet b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/data/vendor_id=1/00000-0-03c9b632-a796-4f56-97b0-a638a6f6d6f4-00001.parquet deleted file mode 100644 index e2179ee7ae0850cbc78e245e6d8a2aaf6b1cd120..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1501 zcma)6O=uHQ5PsQg>Nd94O5c)&1;avPg)}rt6GNy~>7`gGrO;Dro6WwrbtQk@Y${TU zUet;=!AlVjRYY%E@la15iWe0H1rKWVpyEM9sv@3r_AlAe6kPUa-pqX8%$u3FW5>qh z7~#D*hW~7yf9n@914|Bb2<^WNFY8jsIfQ<{U%F!|Ebjz<_37Qi)~4QXzCL}97$m;@ zw&wVAxDWqbzIL^}ssG5xg4%&v);gcDE`0|7`f+Kst*QIucI{!oW&c14ewRn^Z&gpl>| zsJWY!!@<`#Vn>106+LF(s1WjvItr2J@DzkYg2)q`bEI`2o_53)2CI&^+S=?QT`p>5N<~ z*Gk3&hj1I{_En9tsVtEh^)x&MdA1v+f}#jn@{Aw~QxQ(YtkBIt)f#~@%QO5S55|E; zSsN0Mdg{e)XkpthNeDAQy&1q={1-nSL=jTwcemlN=OP5y-BvUtlYP6^%YLB+*l)2W z`?^oWoSpvGs#rnN{4{tB*j=^Xz;5-}UU^xUTQJXmWN{2dRu~ZnZB}npoy1kJL+0){ zG$b?g?OJb)yJn0l0W=cOX;BVsFbk4#rb4BxR5kRPVn|WRHuY*m%JxMiRigz`m9moy zlBtPDrIK7U{p;fh5S`A|G@XJSNNCihER;*L(qhy`ChACMZ2uFN+=_F@duOy-D9c7Y zY=0f@IR`s~Nx24Ru8vOC(c4eV&LmJu&D{BkMHD(-k@dM6HF}GBc%JH2ty~JnWAQ{R z8Scr~v;qmM8KO`U&m@$7C9dWPRnm%*PsTH;RPspfCmJ(|e->@FN|A GANxP(j8c~X diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/data/vendor_id=1/00003-3-ae86257a-5d0b-4c42-9782-f08ec510637e-00001.parquet b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/data/vendor_id=1/00003-3-ae86257a-5d0b-4c42-9782-f08ec510637e-00001.parquet deleted file mode 100644 index 35c6a4918bfa51debd745ddffdb1600aa19f805f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1500 zcma)6O=uHA6rOB1aT{x`(iyT4Ff2A!Ohc12X(+K+L=bx^LcwBfv)P%g3;Ag`o2rz8 zhgu3=^ybA&6+sX|5D^a|v`0}|P!DPap@mZL4~lpYe7l=XwloEo{dx1f_kC~X&6^z> zI3L3Zx8W%M^XtZ^rXc2EC}07hBRAn?T?)O9(C;q`4^6;|j^ihDt4oceKW@B!i#T-k zbL-D)ynws$*7rxxn;WVR4k~3Ht&r8Dyd$5++gtA&tDktf{^I3@SMNON_PX$`n!u0a zcynpxF4r)DLbKTg9{ugz-t_p7qb#^pk( zQZ$AIgj)bRTsBH3Sfu0X6?pRUL7dr%~T0Jr52}k+~I5YLK z)-8fMYsIdkuxw}~ggKD@;l=G_qY3w;FsZ33?ef2uG()aMD#nIRPXFmtCJ z>Xte4-KaaqV>8DsuQ~I1I1DXzjt0AArAm`5gk_+^;0AH)>y}$K<=%L)gvbMqaAAS0 zHMN`l7iiTKmSg%vw5(ia1v#+B%}BYmzV3*mYAjF7Qs2cH z$wXoisVEoB_*y#xMQ2kLO=sW-5f*VI^QGd1G#jy*@fx$*w*SdX?BqG^y*FFUmt>=s zwx>pSorAr}v{ZpJS7Rq@?A<43ZxJY^X0H6wEDD^Pl=Z0!Gr9_TXqxF|tyB!fqOo{1 z5$eoUv^))|X{s<9OUIQSC8p*mQ&NhOOT^O2WTH2hl)Gar#b_#bTpWe->Aul1_>T_3 GfBhesKTO#G diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/data/vendor_id=2/00001-1-2a1bfa65-21d8-4302-ad47-85c00b092e8b-00001.parquet b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/data/vendor_id=2/00001-1-2a1bfa65-21d8-4302-ad47-85c00b092e8b-00001.parquet deleted file mode 100644 index b820af2f93c28430c1970b8e070cac2617d158c8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1499 zcma)6&1(};5P#Whnl)Cd()Y;1f?=g;#WZx2CWcUo;-y$9rO=bL&F;RJx{|MMHWeua zk%B^@NB@AHJb3a0!9x$F2Ze&ri(jRPh)40@AK>g~wloDFo85W8nfcAUnR%JLIj3NR zFW@ZR{&29*OPGVLfCYpuZ^Fx2y7ma6!O!tw8*r?1c<12#;XqI8Z$I`wA`ZpAUVG)) z3wR7~{@QsO?`f}BW?Kkl-$}c?Pd|>|e)+W9+kARD$`$f)G<9We%i~Yr9DZ{2=5OzO zbXuwWiK6n$^!*qF5HE6KQs6+cS4hO1NPP9{TLZq@7ei{uGj&5uI-d|MRPB{6f+`S0 z>d1_kjLdj|LN=bSw?+LqeT*?b?u2|&+QRg*2L^=7AG%?k+k19quxHEpm# zm#PooS5udVP$3~nkR{#wtPd0NoyS>}-sU8na9P7u^%B=W51IRu(Ut7XPom)%FYFjwF+1~( zsD%677(Ke=ges8=m`%Wh!2{&ZH%zx{)9uHwjL372aA|=YMhETrOP)7$rZICl+t%-~ zx|%%V)@18pi^*lVZJ8b2k~6Za8a1d~9?Qs8gVktTF5g*`ZA8h)4Yh9D2jd7XCR^zk zCIdZ?u#8VxYc`hU^^EJ83p~BC{ZGIAaX){&Pxr3YnyM9q9S__?_uzDA+U&rc3%m<~ z_sJV`Iti50uupz>9VKtKRCA@ntkJreT4iS2Xf{$xR>@`asS&MX)M%<&qB^5WDW{L? pN>!swFY3CMS4xFK{<>CB#}rm%w5VMb7vO$|A2bXzI}3m7{{T1LK2`t# diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/data/vendor_id=2/00002-2-22cded08-1e3c-4905-a73c-5e0ea8ed268f-00001.parquet b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/data/vendor_id=2/00002-2-22cded08-1e3c-4905-a73c-5e0ea8ed268f-00001.parquet deleted file mode 100644 index 8d4746fa7fabc25ea4f62f0bf7427318c1fdf616..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1501 zcma)6O=uHQ5PsQg+BH_I()Y+h)VR>JLK?bB6GJFf=|x&BRR{>yHv99ouI8uRY${TU zdQdBZO2vbUmmb8UD20MI5us2}Jc)(s!Gq|fhgfY!Jl70dB`+%zc{yldGJnd#fOI0&ir*E z1aSi2TwnUt-V~y!`TT|-WnV-u1Rw;zm*GPk1Cn)lM9lESH7^aeyJlDPm|0EJj`E?} z$D`)1|2VaLARrtAQdjhteItCxH{vM5gu`PH4)I~4;=Cih@!@etTx783i1U8i0icBw z0aKC-6bBJzJ5gxI3@xc;V?tMRjIC)k4SmX5v(QNJ*i~Drdc~AV3eB}390cu7O)}_& zR4i9ZW{yL+4Rre|X4z7f)CuhpJOyd86QzRTFl0#>KFp6rxiDt=P7bQp2#i_6kO2b5 z{zh3F5<%VdVmGv~ZI~p4p$+6`0C$jYemsDpYMtNRhQpqZs=)5Hq9IxATeV*H8!f>8 zl(pF3zyxx3`kSj#3z8<|;L&e))qVqet;hDt%lfDV6Y`bCaTHyIK@Hlh-l{r@YhZ`W z-EnA0R_0r^-WboV7z+U_^LaRHZFY<%U8;r3kUY5c!-T;N<|fFfUJ0AuCi&qpFxfOt4Kox+-LQV}hpBf?5%> zXJ-XVBgTZ1RJ8o-;|LIgPFHnLgMzoBoP*uz%ls-k&WR)suMUj(YCY4GakyBEyNYhkJ%ZK?fIG^4db-|Bx I6n^ag0JwikTL1t6 diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-0b944f80-1326-447a-97fd-f4827081b3c5.metadata.json b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-0b944f80-1326-447a-97fd-f4827081b3c5.metadata.json deleted file mode 100644 index ffbf8a39..00000000 --- a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-0b944f80-1326-447a-97fd-f4827081b3c5.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"format-version":1,"table-uuid":"df838b92-0b32-465d-a44e-d39936e538b7","location":"/home/iceberg/warehouse/nyc/taxis","last-updated-ms":1662532818843,"last-column-id":5,"schema":{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]},"schemas":[{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]}],"current-schema-id":0,"partition-spec":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}],"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{"owner":"root"},"current-snapshot-id":638933773299822130,"snapshots":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843,"manifest-list":"/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro","summary":{"operation":"append","total-equality-deletes":"0","total-position-deletes":"0","spark.app.id":"local-1662532784305","added-records":"4","total-data-files":"4","total-files-size":"6001","added-files-size":"6001","changed-partition-count":"2","added-data-files":"4","total-records":"4","total-delete-files":"0"},"schema-id":0}],"snapshot-log":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843}],"metadata-log":[{"metadata-file":"/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json","timestamp-ms":1662532805245}],"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0} \ No newline at end of file diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-1a907b10-fcab-4be8-903c-25bc0974156b.metadata.json b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-1a907b10-fcab-4be8-903c-25bc0974156b.metadata.json deleted file mode 100644 index 68a6146b..00000000 --- a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-1a907b10-fcab-4be8-903c-25bc0974156b.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"format-version":1,"table-uuid":"df838b92-0b32-465d-a44e-d39936e538b7","location":"/home/iceberg/warehouse/nyc/taxis","last-updated-ms":1662532818843,"last-column-id":5,"schema":{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]},"schemas":[{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]}],"current-schema-id":0,"partition-spec":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}],"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{"owner":"root"},"current-snapshot-id":638933773299822130,"snapshots":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843,"manifest-list":"/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro","summary":{"operation":"append","total-data-files":"4","added-data-files":"4","total-delete-files":"0","total-position-deletes":"0","added-records":"4","added-files-size":"6001","total-records":"4","total-equality-deletes":"0","total-files-size":"6001","spark.app.id":"local-1662532784305","changed-partition-count":"2"},"schema-id":0}],"snapshot-log":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843}],"metadata-log":[{"metadata-file":"/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json","timestamp-ms":1662532805245}],"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0} \ No newline at end of file diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-1c785339-c427-4232-ab45-704a663ff72b.metadata.json b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-1c785339-c427-4232-ab45-704a663ff72b.metadata.json deleted file mode 100644 index 64db4f32..00000000 --- a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-1c785339-c427-4232-ab45-704a663ff72b.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"format-version":1,"table-uuid":"df838b92-0b32-465d-a44e-d39936e538b7","location":"/home/iceberg/warehouse/nyc/taxis","last-updated-ms":1662532818843,"last-column-id":5,"schema":{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]},"schemas":[{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]}],"current-schema-id":0,"partition-spec":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}],"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{"owner":"root"},"current-snapshot-id":638933773299822130,"snapshots":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843,"manifest-list":"/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro","summary":{"operation":"append","changed-partition-count":"2","spark.app.id":"local-1662532784305","total-files-size":"6001","total-records":"4","added-files-size":"6001","added-data-files":"4","total-data-files":"4","total-position-deletes":"0","total-delete-files":"0","added-records":"4","total-equality-deletes":"0"},"schema-id":0}],"snapshot-log":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843}],"metadata-log":[{"metadata-file":"/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json","timestamp-ms":1662532805245}],"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0} \ No newline at end of file diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-2b717789-530f-4120-9563-ccad06de4098.metadata.json b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-2b717789-530f-4120-9563-ccad06de4098.metadata.json deleted file mode 100644 index d64ba9e4..00000000 --- a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-2b717789-530f-4120-9563-ccad06de4098.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"format-version":1,"table-uuid":"df838b92-0b32-465d-a44e-d39936e538b7","location":"/home/iceberg/warehouse/nyc/taxis","last-updated-ms":1662532818843,"last-column-id":5,"schema":{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]},"schemas":[{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]}],"current-schema-id":0,"partition-spec":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}],"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{"owner":"root"},"current-snapshot-id":638933773299822130,"snapshots":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843,"manifest-list":"/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro","summary":{"operation":"append","total-equality-deletes":"0","total-files-size":"6001","total-records":"4","total-data-files":"4","added-files-size":"6001","total-position-deletes":"0","added-data-files":"4","added-records":"4","spark.app.id":"local-1662532784305","changed-partition-count":"2","total-delete-files":"0"},"schema-id":0}],"snapshot-log":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843}],"metadata-log":[{"metadata-file":"/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json","timestamp-ms":1662532805245}],"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0} \ No newline at end of file diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-300c39ad-0663-4f3c-9523-e338aebb25ef.metadata.json b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-300c39ad-0663-4f3c-9523-e338aebb25ef.metadata.json deleted file mode 100644 index 52afc021..00000000 --- a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-300c39ad-0663-4f3c-9523-e338aebb25ef.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"format-version":1,"table-uuid":"df838b92-0b32-465d-a44e-d39936e538b7","location":"/home/iceberg/warehouse/nyc/taxis","last-updated-ms":1662532818843,"last-column-id":5,"schema":{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]},"schemas":[{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]}],"current-schema-id":0,"partition-spec":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}],"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{"owner":"root"},"current-snapshot-id":638933773299822130,"snapshots":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843,"manifest-list":"/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro","summary":{"operation":"append","spark.app.id":"local-1662532784305","added-records":"4","total-data-files":"4","total-delete-files":"0","total-files-size":"6001","total-position-deletes":"0","added-files-size":"6001","changed-partition-count":"2","total-equality-deletes":"0","added-data-files":"4","total-records":"4"},"schema-id":0}],"snapshot-log":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843}],"metadata-log":[{"metadata-file":"/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json","timestamp-ms":1662532805245}],"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0} \ No newline at end of file diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-5b83190d-8882-44ca-9b1d-92c7297d3f2e.metadata.json b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-5b83190d-8882-44ca-9b1d-92c7297d3f2e.metadata.json deleted file mode 100644 index bd7d7ce1..00000000 --- a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-5b83190d-8882-44ca-9b1d-92c7297d3f2e.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"format-version":1,"table-uuid":"df838b92-0b32-465d-a44e-d39936e538b7","location":"/home/iceberg/warehouse/nyc/taxis","last-updated-ms":1662532818843,"last-column-id":5,"schema":{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]},"schemas":[{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]}],"current-schema-id":0,"partition-spec":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}],"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{"owner":"root"},"current-snapshot-id":638933773299822130,"snapshots":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843,"manifest-list":"/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro","summary":{"operation":"append","total-records":"4","added-records":"4","added-files-size":"6001","total-files-size":"6001","total-position-deletes":"0","spark.app.id":"local-1662532784305","added-data-files":"4","total-delete-files":"0","total-equality-deletes":"0","total-data-files":"4","changed-partition-count":"2"},"schema-id":0}],"snapshot-log":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843}],"metadata-log":[{"metadata-file":"/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json","timestamp-ms":1662532805245}],"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0} \ No newline at end of file diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-730360ca-44bb-449f-a942-d936fe75932e.metadata.json b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-730360ca-44bb-449f-a942-d936fe75932e.metadata.json deleted file mode 100644 index d162992a..00000000 --- a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-730360ca-44bb-449f-a942-d936fe75932e.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"format-version":1,"table-uuid":"df838b92-0b32-465d-a44e-d39936e538b7","location":"/home/iceberg/warehouse/nyc/taxis","last-updated-ms":1662532818843,"last-column-id":5,"schema":{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]},"schemas":[{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]}],"current-schema-id":0,"partition-spec":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}],"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{"owner":"root"},"current-snapshot-id":638933773299822130,"snapshots":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843,"manifest-list":"/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro","summary":{"operation":"append","total-data-files":"4","total-position-deletes":"0","total-equality-deletes":"0","added-files-size":"6001","changed-partition-count":"2","total-files-size":"6001","total-records":"4","added-records":"4","added-data-files":"4","total-delete-files":"0","spark.app.id":"local-1662532784305"},"schema-id":0}],"snapshot-log":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843}],"metadata-log":[{"metadata-file":"/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json","timestamp-ms":1662532805245}],"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0} \ No newline at end of file diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-9b3fb15d-c882-4021-b385-9d74cb3ecb3b.metadata.json b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-9b3fb15d-c882-4021-b385-9d74cb3ecb3b.metadata.json deleted file mode 100644 index 977b82fd..00000000 --- a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-9b3fb15d-c882-4021-b385-9d74cb3ecb3b.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"format-version":1,"table-uuid":"df838b92-0b32-465d-a44e-d39936e538b7","location":"/home/iceberg/warehouse/nyc/taxis","last-updated-ms":1662532818843,"last-column-id":5,"schema":{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]},"schemas":[{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]}],"current-schema-id":0,"partition-spec":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}],"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{"owner":"root"},"current-snapshot-id":638933773299822130,"snapshots":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843,"manifest-list":"/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro","summary":{"operation":"append","added-data-files":"4","added-records":"4","total-records":"4","total-delete-files":"0","added-files-size":"6001","total-equality-deletes":"0","total-files-size":"6001","spark.app.id":"local-1662532784305","changed-partition-count":"2","total-data-files":"4","total-position-deletes":"0"},"schema-id":0}],"snapshot-log":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843}],"metadata-log":[{"metadata-file":"/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json","timestamp-ms":1662532805245}],"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0} \ No newline at end of file diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-a5662485-3c04-49a0-af8a-a70c5d3fcc16.metadata.json b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-a5662485-3c04-49a0-af8a-a70c5d3fcc16.metadata.json deleted file mode 100644 index 5962d56e..00000000 --- a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-a5662485-3c04-49a0-af8a-a70c5d3fcc16.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"format-version":1,"table-uuid":"df838b92-0b32-465d-a44e-d39936e538b7","location":"/home/iceberg/warehouse/nyc/taxis","last-updated-ms":1662532818843,"last-column-id":5,"schema":{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]},"schemas":[{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]}],"current-schema-id":0,"partition-spec":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}],"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{"owner":"root"},"current-snapshot-id":638933773299822130,"snapshots":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843,"manifest-list":"/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro","summary":{"operation":"append","added-data-files":"4","total-equality-deletes":"0","spark.app.id":"local-1662532784305","total-records":"4","total-data-files":"4","added-records":"4","added-files-size":"6001","changed-partition-count":"2","total-position-deletes":"0","total-delete-files":"0","total-files-size":"6001"},"schema-id":0}],"snapshot-log":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843}],"metadata-log":[{"metadata-file":"/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json","timestamp-ms":1662532805245}],"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0} \ No newline at end of file diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-bd4a6f4b-c07f-4405-90dc-5bd2a83501f0.metadata.json b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-bd4a6f4b-c07f-4405-90dc-5bd2a83501f0.metadata.json deleted file mode 100644 index 93d49570..00000000 --- a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-bd4a6f4b-c07f-4405-90dc-5bd2a83501f0.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"format-version":1,"table-uuid":"df838b92-0b32-465d-a44e-d39936e538b7","location":"/home/iceberg/warehouse/nyc/taxis","last-updated-ms":1662532818843,"last-column-id":5,"schema":{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]},"schemas":[{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]}],"current-schema-id":0,"partition-spec":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}],"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{"owner":"root"},"current-snapshot-id":638933773299822130,"snapshots":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843,"manifest-list":"/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro","summary":{"operation":"append","total-equality-deletes":"0","changed-partition-count":"2","added-data-files":"4","spark.app.id":"local-1662532784305","added-records":"4","added-files-size":"6001","total-data-files":"4","total-records":"4","total-files-size":"6001","total-delete-files":"0","total-position-deletes":"0"},"schema-id":0}],"snapshot-log":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843}],"metadata-log":[{"metadata-file":"/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json","timestamp-ms":1662532805245}],"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0} \ No newline at end of file diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-cd9e2028-df97-4774-ac2e-9d11002033be.metadata.json b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-cd9e2028-df97-4774-ac2e-9d11002033be.metadata.json deleted file mode 100644 index 410a587a..00000000 --- a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-cd9e2028-df97-4774-ac2e-9d11002033be.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"format-version":1,"table-uuid":"df838b92-0b32-465d-a44e-d39936e538b7","location":"/home/iceberg/warehouse/nyc/taxis","last-updated-ms":1662532818843,"last-column-id":5,"schema":{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]},"schemas":[{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]}],"current-schema-id":0,"partition-spec":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}],"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{"owner":"root"},"current-snapshot-id":638933773299822130,"snapshots":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843,"manifest-list":"/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro","summary":{"operation":"append","total-records":"4","changed-partition-count":"2","total-delete-files":"0","added-data-files":"4","total-position-deletes":"0","total-equality-deletes":"0","total-data-files":"4","added-records":"4","added-files-size":"6001","spark.app.id":"local-1662532784305","total-files-size":"6001"},"schema-id":0}],"snapshot-log":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843}],"metadata-log":[{"metadata-file":"/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json","timestamp-ms":1662532805245}],"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0} \ No newline at end of file diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-e9cb38f8-720c-458b-8cf4-7411ce9f5eea.metadata.json b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-e9cb38f8-720c-458b-8cf4-7411ce9f5eea.metadata.json deleted file mode 100644 index 98756ef1..00000000 --- a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-e9cb38f8-720c-458b-8cf4-7411ce9f5eea.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"format-version":1,"table-uuid":"df838b92-0b32-465d-a44e-d39936e538b7","location":"/home/iceberg/warehouse/nyc/taxis","last-updated-ms":1662532818843,"last-column-id":5,"schema":{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]},"schemas":[{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]}],"current-schema-id":0,"partition-spec":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}],"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{"owner":"root"},"current-snapshot-id":638933773299822130,"snapshots":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843,"manifest-list":"/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro","summary":{"operation":"append","changed-partition-count":"2","added-data-files":"4","total-position-deletes":"0","added-records":"4","total-records":"4","total-files-size":"6001","total-data-files":"4","total-delete-files":"0","spark.app.id":"local-1662532784305","added-files-size":"6001","total-equality-deletes":"0"},"schema-id":0}],"snapshot-log":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843}],"metadata-log":[{"metadata-file":"/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json","timestamp-ms":1662532805245}],"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0} \ No newline at end of file diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-f44e4bfe-c5e9-4483-8a97-bedce302e5d0.metadata.json b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-f44e4bfe-c5e9-4483-8a97-bedce302e5d0.metadata.json deleted file mode 100644 index 5d0cca3a..00000000 --- a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-f44e4bfe-c5e9-4483-8a97-bedce302e5d0.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"format-version":1,"table-uuid":"df838b92-0b32-465d-a44e-d39936e538b7","location":"/home/iceberg/warehouse/nyc/taxis","last-updated-ms":1662532818843,"last-column-id":5,"schema":{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]},"schemas":[{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]}],"current-schema-id":0,"partition-spec":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}],"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{"owner":"root"},"current-snapshot-id":638933773299822130,"snapshots":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843,"manifest-list":"/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro","summary":{"operation":"append","added-records":"4","total-files-size":"6001","total-records":"4","changed-partition-count":"2","added-files-size":"6001","spark.app.id":"local-1662532784305","total-data-files":"4","total-position-deletes":"0","total-delete-files":"0","total-equality-deletes":"0","added-data-files":"4"},"schema-id":0}],"snapshot-log":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843}],"metadata-log":[{"metadata-file":"/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json","timestamp-ms":1662532805245}],"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0} \ No newline at end of file diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-fb3797e1-6a8a-4bb7-9d18-e9ea5bb998d5.metadata.json b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-fb3797e1-6a8a-4bb7-9d18-e9ea5bb998d5.metadata.json deleted file mode 100644 index 540f3eee..00000000 --- a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/0-fb3797e1-6a8a-4bb7-9d18-e9ea5bb998d5.metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"format-version":1,"table-uuid":"df838b92-0b32-465d-a44e-d39936e538b7","location":"/home/iceberg/warehouse/nyc/taxis","last-updated-ms":1662532818843,"last-column-id":5,"schema":{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]},"schemas":[{"schema-id":0,"type":"struct","fields":[{"id":1,"name":"vendor_id","required":false,"type":"long"},{"id":2,"name":"trip_id","required":false,"type":"long"},{"id":3,"name":"trip_distance","required":false,"type":"float"},{"id":4,"name":"fare_amount","required":false,"type":"double"},{"id":5,"name":"store_and_fwd_flag","required":false,"type":"string"}]}],"current-schema-id":0,"partition-spec":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}],"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"name":"vendor_id_partition","transform":"identity"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{"owner":"root"},"current-snapshot-id":638933773299822130,"snapshots":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843,"manifest-list":"/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro","summary":{"operation":"append","total-position-deletes":"0","total-data-files":"4","total-files-size":"6001","spark.app.id":"local-1662532784305","changed-partition-count":"2","added-records":"4","total-equality-deletes":"0","added-data-files":"4","total-delete-files":"0","added-files-size":"6001","total-records":"4"},"schema-id":0}],"snapshot-log":[{"snapshot-id":638933773299822130,"timestamp-ms":1662532818843}],"metadata-log":[{"metadata-file":"/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json","timestamp-ms":1662532805245}],"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0} \ No newline at end of file diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/7e6760f0-4f6c-4b23-b907-0a5a174e3863-m0.avro b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/7e6760f0-4f6c-4b23-b907-0a5a174e3863-m0.avro deleted file mode 100644 index 87bc1e2691263f44516db816e009dd4812f00106..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6447 zcmb_g32YQq7}knFz#v*QDpbee(2xaoW=bz&5TZ@cS^_P21vhVJ-?n3CXO=nIVy##3 z08Et@G$d-|kVdo+MNxyGVkAgnVnS35K`dfrG#Upz=jDrF z?VmMJ+HN%+4`@%|UeR!M9pq_ZmPEt^P9j%##5(dkW07i-I86!Upjc7^NPNS2W3c3$ zeiPwX8ab_j22ySW9&o`_p_&>@6*-z|2#XfZ6FM}fRj6+gC}hMci%^P2InSuxq-muf z+rDpM&Gleg@XiPXnmU1`BO;iVO8y9GWG!_uk>{=O@xuv6BhCdj^2aYlb0ffa%b;@+ zg`k&oXFDi!FnE;|Q!{m)DGr)rAKykd6~~rfCI#=XEog?|v;s0-UisJ+PBlf?)*iqy z&4|U9Ihd=yECbD^gLJA*AjgS%|2VI^lG7d6O|>mnSUE2{lap0V-Ax%(n0e%vcaR=r zS%`F!=HL{}u;6*P_1YwZBfk*k9H(C?+$t#YbfvOgu zlB1Q_gCFLe-|! zmI#(VSD~&1idryQy$&u|YzjX-bh86nu{dB3gv%c|=P}ddDpF_FB|s-vsQrJO+xt&!> zqUT(2-`r}b*0MAuUmz#=asph>@q&3#_-SOPbKxl%qzy{&0Fz=RvQCEh!7@4x; zD8*OJ1Xjm@cRF(L1dnr0YsZ!i+_6j&10FfePGc2fEsIXjR1CbVC9@qIpBzw>;?^-( z(0sb@uLf+ZfrXB(VkBD9)Y(g%owXz|(W2HkelK96yw$U#q4zttPDWV-lxk*{gxu=F z)V|Qt5X2f~D=W1StP922s4sGTVD{La8zdYGDz~*_0AdL7O9a!GD_2XpnLliO4($m{ z1)np&CB40@@ZP|Ta62~H^}?jS7*lyS2xf(?;xL?Y><)#@FfBDf(}cDNCZ2_9kR-y> zBO(k*Z%#?o^dHz>y*-@X0hw1|!@Cmdd_29k%ml$4b~+E{A~`>cp*Dzt>SgSd87M9W zP((Oqju>PpyrGGqFc}Yml31nZ;@yLcOoqPC7a6qZ`9^jYOAf+Npe!;&IR_ygWE2V% zQl2OjNZt8IaZdUg#3i%t3o+F}#w1tejY$UV{FvgJf#`QI`d}I}a?Z}-i6wKNwS9;~ zGR;M`jGVt^6s~1Tcb}(xf%^)(#-JW}c9+L}F>t^eF$HNJGsAALFr&+3Czy^pe3yms z7@~5)0~j%x9p68}6lP^K2jvUgt7Ic0%pLi(?L`|8e1 zzqz1qMfI=4E?@ED!OpT-U+fd4c>2`I#V01MJ^jT^R~>n0+x&Ze?%(;*q23P`9BO`O z=AG-dZ2Nb_hOJfickSGNq;CJ2Cu&~*<)4L>%Rkt*BQdah@!Il7=dUa~fL=;QHVM6J z=4?`jYrU6^jU0bWo3gIw<$5>>ey@RliTm!JN>>lrv9;I zA2px@|mArAxzPVM_v#ZALSh@b))3<&9cI4E94;;mBRt#J>rQ(#{wP#mt#f+Vu zJ4Q_2b8ybd_hw7$XEctapSPg7s`8AubKLii{jsY*)rwp z`Rk7M99Y%Uw7hcwrW2jBZWZ3Y`k3?Hh!f-cTEZ#&AA`(?00000 diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/fb072c92-a02b-11e9-ae9c-1bb7bc9eca94.metadata.json b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/fb072c92-a02b-11e9-ae9c-1bb7bc9eca94.metadata.json deleted file mode 100644 index 9ba3f40b..00000000 --- a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/fb072c92-a02b-11e9-ae9c-1bb7bc9eca94.metadata.json +++ /dev/null @@ -1,128 +0,0 @@ -{ - "format-version" : 1, - "table-uuid" : "df838b92-0b32-465d-a44e-d39936e538b7", - "location" : "/home/iceberg/warehouse/nyc/taxis", - "last-updated-ms" : 1662532818843, - "last-column-id" : 5, - "schema" : { - "type" : "struct", - "schema-id" : 0, - "fields" : [ { - "id" : 1, - "name" : "vendor_id", - "required" : false, - "type" : "long" - }, { - "id" : 2, - "name" : "trip_id", - "required" : false, - "type" : "long" - }, { - "id" : 3, - "name" : "trip_distance", - "required" : false, - "type" : "float" - }, { - "id" : 4, - "name" : "fare_amount", - "required" : false, - "type" : "double" - }, { - "id" : 5, - "name" : "store_and_fwd_flag", - "required" : false, - "type" : "string" - } ] - }, - "current-schema-id" : 0, - "schemas" : [ { - "type" : "struct", - "schema-id" : 0, - "fields" : [ { - "id" : 1, - "name" : "vendor_id", - "required" : false, - "type" : "long" - }, { - "id" : 2, - "name" : "trip_id", - "required" : false, - "type" : "long" - }, { - "id" : 3, - "name" : "trip_distance", - "required" : false, - "type" : "float" - }, { - "id" : 4, - "name" : "fare_amount", - "required" : false, - "type" : "double" - }, { - "id" : 5, - "name" : "store_and_fwd_flag", - "required" : false, - "type" : "string" - } ] - } ], - "partition-spec" : [ { - "name" : "vendor_id_partition", - "transform" : "identity", - "source-id" : 1, - "field-id" : 1000 - } ], - "default-spec-id" : 0, - "partition-specs" : [ { - "spec-id" : 0, - "fields" : [ { - "name" : "vendor_id_partition", - "transform" : "identity", - "source-id" : 1, - "field-id" : 1000 - } ] - } ], - "last-partition-id" : 1000, - "default-sort-order-id" : 0, - "sort-orders" : [ { - "order-id" : 0, - "fields" : [ ] - } ], - "properties" : { - "owner" : "root" - }, - "current-snapshot-id" : 638933773299822130, - "refs" : { - "main" : { - "snapshot-id" : 638933773299822130, - "type" : "branch" - } - }, - "snapshots" : [ { - "snapshot-id" : 638933773299822130, - "timestamp-ms" : 1662532818843, - "summary" : { - "operation" : "append", - "spark.app.id" : "local-1662532784305", - "added-data-files" : "4", - "added-records" : "4", - "added-files-size" : "6001", - "changed-partition-count" : "2", - "total-records" : "4", - "total-files-size" : "6001", - "total-data-files" : "4", - "total-delete-files" : "0", - "total-position-deletes" : "0", - "total-equality-deletes" : "0" - }, - "manifest-list" : "/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro", - "schema-id" : 0 - } ], - "snapshot-log" : [ { - "timestamp-ms" : 1662532818843, - "snapshot-id" : 638933773299822130 - } ], - "metadata-log" : [ { - "timestamp-ms" : 1662532805245, - "metadata-file" : "/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json" - } ] -} \ No newline at end of file diff --git a/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro b/iceberg-tests/nyc_taxis/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro deleted file mode 100644 index 7b448b5a34cd180f28d02abcd0f0631a5e76b2b0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3766 zcmbW3O>Y}T7{{B40|&$jRfra8v??c3@Jn9o2vq`=N|h>sNCA^5+KhK+vlFj(+}YU# zQ(1rnS5%t-X;SJQtIDc4aO8qi3HS_LIKnqTRS(SU%gn6T8`in`W#;)k&&>0G{_|dY zZ{daWaG#Pl0?S2R`1HbUgY|pJST`uL2(^tR!-rk*y$gNppn#bU_7MNs!N{`%V}0v( zZbc6=m#;_y1&m^Umw&g3#pB!{7G#+C;KL8Dg9FT5aBTxbbQ$$BwvF|bN@cK=kL@9U zSC8%yE%r^qpa(>PAi$p?fPK*JGZduhCV{J3;2xw5i+s$Whb)tmW#ZUpGV2ZgC%AqU zgaNVvBjA8yHl1%x3kz)<*=FFwUf>d@hut##(DQiUDD&;~ST>TS0>WI#Koo(0L?AtR zkqmNR=rLnm#7OQSDXN~M+mONNwgS^4VN^_fB{Bt3q0HcIkupF&qkWLBKZUof@zAF@ zVEmHJIg#%s`^r;SG^&j}#P})C3B4L`E6-fjn6h0&Pi0S2DhE;u`}}X-AzdyI`TMk@ z=t`x*m*s_I5uZWq2WHYKiICcacnJEU4I(Aoq#9?*v3&~nvro?1_3 zO*_bV>QHh}sQB#Ai6t-Amr#6FpKns}ZJA%sOOX{?#75SO zyoEf-(D`(R^BcRgT5mM#^|iHnt=VkUYSnt>BEPOKWS9354Y-LcRxe@;wGrJ-4E5(% zr2M%LhL&)Y*lME=sAS}y*k0ap6GNrwR){I$=Ey^#bp9A~7_8`(m{QGCq_q}s%2};K z)FeVqBh6$BApdW=3Q(7Txbf4gitSbb9Own=xH4swv0VjRo&wHro+7GVSKetAVnrgP z#Lh^OdebdkfmS6b)z%p~snBL))+9zYff&#Q$0Wt#@dpK8HZ?8f51Ok=S!q5Rw|G;c zmB+!5SPimfzbmMX5eFYcmo(x+Ia%2iFF8Sbb8XRjnPBXTaOf~yH6)tYgrQt_ow zHLFqxHFHv(&QC}ssspwMm$)LuQzRXm+>xFakH>Qto*JFMx4&y$8NN2Sw(!@Fi~lZc z{q}G?9G^TI-@SKyxOw>4yx%%J9-5o?9*sY~J26MC;U|;v$>!*2)EYe`_xBv%x#K+O zjGXT$osn%%PHepMxO2xnc3pq8=Rs$3(s}Sh>!ZWZ-~9T^fByL9+gJbGfA-7kzd!l) Z$+@p~-uYs