From 7e7a57add695f75b23c48af9c926f4e7129f465d Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Fri, 5 Jan 2024 11:14:30 +0000 Subject: [PATCH] Prepare object_store 0.9.0 --- Cargo.toml | 3 ++ datafusion-cli/Cargo.lock | 35 ++++++++++++------- datafusion-cli/Cargo.toml | 3 ++ .../core/src/datasource/physical_plan/csv.rs | 2 +- .../core/src/datasource/physical_plan/json.rs | 2 +- .../core/src/datasource/physical_plan/mod.rs | 6 ++-- 6 files changed, 32 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a87923b6a1a0..6e592336a996 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,3 +92,6 @@ opt-level = 3 overflow-checks = false panic = 'unwind' rpath = false + +[patch.crates-io] +object_store = { git = "https://github.com/apache/arrow-rs.git", rev = "f7101ec3a2b37c436f4554c28fa2d0a05de533ff" } diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 252b00ca0adc..bbdfa73c5dee 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -2296,8 +2296,7 @@ dependencies = [ [[package]] name = "object_store" version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2524735495ea1268be33d200e1ee97455096a0846295a21548cd2f3541de7050" +source = "git+https://github.com/apache/arrow-rs.git?rev=f7101ec3a2b37c436f4554c28fa2d0a05de533ff#f7101ec3a2b37c436f4554c28fa2d0a05de533ff" dependencies = [ "async-trait", "base64", @@ -2306,14 +2305,14 @@ dependencies = [ "futures", "humantime", "hyper", - "itertools 0.11.0", + "itertools 0.12.0", "parking_lot", "percent-encoding", "quick-xml", "rand", "reqwest", "ring 0.17.7", - "rustls-pemfile", + "rustls-pemfile 2.0.0", "serde", "serde_json", "snafu", @@ -2740,7 +2739,8 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustls 0.21.10", - "rustls-pemfile", + "rustls-native-certs", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", @@ -2754,7 +2754,6 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", "winreg", ] @@ -2878,7 +2877,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "schannel", "security-framework", ] @@ -2892,6 +2891,22 @@ dependencies = [ "base64", ] +[[package]] +name = "rustls-pemfile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" +dependencies = [ + "base64", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -3768,12 +3783,6 @@ dependencies = [ "untrusted 0.9.0", ] -[[package]] -name = "webpki-roots" -version = "0.25.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" - [[package]] name = "winapi" version = "0.3.9" diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index eab7c8e0d1f8..65a1e26bf6f1 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -53,3 +53,6 @@ assert_cmd = "2.0" ctor = "0.2.0" predicates = "3.0" rstest = "0.17" + +[patch.crates-io] +object_store = { git = "https://github.com/apache/arrow-rs.git", rev = "f7101ec3a2b37c436f4554c28fa2d0a05de533ff" } diff --git a/datafusion/core/src/datasource/physical_plan/csv.rs b/datafusion/core/src/datasource/physical_plan/csv.rs index b28bc7d56688..a818c572f7f5 100644 --- a/datafusion/core/src/datasource/physical_plan/csv.rs +++ b/datafusion/core/src/datasource/physical_plan/csv.rs @@ -375,7 +375,7 @@ impl FileOpener for CsvOpener { let range = match calculated_range { RangeCalculation::Range(None) => None, - RangeCalculation::Range(Some(range)) => Some(range), + RangeCalculation::Range(Some(range)) => Some(range.into()), RangeCalculation::TerminateEarly => { return Ok( futures::stream::poll_fn(move |_| Poll::Ready(None)).boxed() diff --git a/datafusion/core/src/datasource/physical_plan/json.rs b/datafusion/core/src/datasource/physical_plan/json.rs index 529632dab85a..a8a371fed91e 100644 --- a/datafusion/core/src/datasource/physical_plan/json.rs +++ b/datafusion/core/src/datasource/physical_plan/json.rs @@ -239,7 +239,7 @@ impl FileOpener for JsonOpener { let range = match calculated_range { RangeCalculation::Range(None) => None, - RangeCalculation::Range(Some(range)) => Some(range), + RangeCalculation::Range(Some(range)) => Some(range.into()), RangeCalculation::TerminateEarly => { return Ok( futures::stream::poll_fn(move |_| Poll::Ready(None)).boxed() diff --git a/datafusion/core/src/datasource/physical_plan/mod.rs b/datafusion/core/src/datasource/physical_plan/mod.rs index d7be017a1868..24155d3fd167 100644 --- a/datafusion/core/src/datasource/physical_plan/mod.rs +++ b/datafusion/core/src/datasource/physical_plan/mod.rs @@ -75,7 +75,7 @@ use datafusion_physical_plan::ExecutionPlan; use log::debug; use object_store::ObjectMeta; -use object_store::{path::Path, GetOptions, ObjectStore}; +use object_store::{path::Path, GetOptions, GetRange, ObjectStore}; /// The base configurations to provide when creating a physical plan for /// writing to any given file format. @@ -604,10 +604,8 @@ async fn find_first_newline( start: usize, end: usize, ) -> Result { - let range = Some(Range { start, end }); - let options = GetOptions { - range, + range: Some(GetRange::Bounded(start..end)), ..Default::default() };