From 9eb7244c30e39d8b24cac67b50a56619be66b399 Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:51:36 +0100 Subject: [PATCH 1/6] fix: conver to ascii lowercase, filter incorrect keys out --- crates/polars-io/src/cloud/options.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/crates/polars-io/src/cloud/options.rs b/crates/polars-io/src/cloud/options.rs index ea5376b6865f..b7d8a2e19965 100644 --- a/crates/polars-io/src/cloud/options.rs +++ b/crates/polars-io/src/cloud/options.rs @@ -120,16 +120,13 @@ fn parsed_untyped_config, impl Into>>() + .collect::>()) } #[derive(PartialEq)] From 053a00d963e211cc7788ceb12439d047efbdedcb Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:55:36 +0100 Subject: [PATCH 2/6] chore: fmt --- crates/polars-io/src/cloud/options.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/polars-io/src/cloud/options.rs b/crates/polars-io/src/cloud/options.rs index b7d8a2e19965..45a57ea2d9f4 100644 --- a/crates/polars-io/src/cloud/options.rs +++ b/crates/polars-io/src/cloud/options.rs @@ -123,7 +123,8 @@ where Ok(config .into_iter() .filter_map(|(key, val)| { - T::from_str(key.as_ref().to_ascii_lowercase().as_str()).ok() + T::from_str(key.as_ref().to_ascii_lowercase().as_str()) + .ok() .map(|typed_key| (typed_key, val.into())) }) .collect::>()) From fc512336dd8e0f1d3fc738e99e96e18e6184027b Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Sun, 24 Nov 2024 11:17:51 +0100 Subject: [PATCH 3/6] chore: add test --- crates/polars-io/src/cloud/options.rs | 39 ++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/crates/polars-io/src/cloud/options.rs b/crates/polars-io/src/cloud/options.rs index 45a57ea2d9f4..145bfc6909e6 100644 --- a/crates/polars-io/src/cloud/options.rs +++ b/crates/polars-io/src/cloud/options.rs @@ -605,7 +605,9 @@ impl CloudOptions { #[cfg(feature = "cloud")] #[cfg(test)] mod tests { - use super::parse_url; + use hashbrown::HashMap; + + use super::{parse_url, parsed_untyped_config}; #[test] fn test_parse_url() { @@ -680,4 +682,39 @@ mod tests { ); } } + #[cfg(feature = "aws")] + #[test] + fn test_parse_untyped_config() { + use object_store::aws::AmazonS3ConfigKey; + + let aws_config = [ + ("aws_secret_access_key", "a_key"), + ("aws_s3_allow_unsafe_rename", "true"), + ] + .into_iter() + .collect::>(); + let aws_keys = parsed_untyped_config::(aws_config) + .expect("Parsing keys shouldn't have thrown an error"); + + assert_eq!( + aws_keys.get(0).unwrap().0, + AmazonS3ConfigKey::SecretAccessKey + ); + assert_eq!(aws_keys.len(), 1); + + let aws_config = [ + ("AWS_SECRET_ACCESS_KEY", "a_key"), + ("aws_s3_allow_unsafe_rename", "true"), + ] + .into_iter() + .collect::>(); + let aws_keys = parsed_untyped_config::(aws_config) + .expect("Parsing keys shouldn't have thrown an error"); + + assert_eq!( + aws_keys.get(0).unwrap().0, + AmazonS3ConfigKey::SecretAccessKey + ); + assert_eq!(aws_keys.len(), 1); + } } From 33663fa5e396c03d439434ab72ef5fd731e5d0ea Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Sun, 24 Nov 2024 11:22:48 +0100 Subject: [PATCH 4/6] chore: clipppy --- crates/polars-io/src/cloud/options.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/polars-io/src/cloud/options.rs b/crates/polars-io/src/cloud/options.rs index 145bfc6909e6..dd800185ce53 100644 --- a/crates/polars-io/src/cloud/options.rs +++ b/crates/polars-io/src/cloud/options.rs @@ -697,7 +697,7 @@ mod tests { .expect("Parsing keys shouldn't have thrown an error"); assert_eq!( - aws_keys.get(0).unwrap().0, + aws_keys.first().unwrap().0, AmazonS3ConfigKey::SecretAccessKey ); assert_eq!(aws_keys.len(), 1); @@ -712,7 +712,7 @@ mod tests { .expect("Parsing keys shouldn't have thrown an error"); assert_eq!( - aws_keys.get(0).unwrap().0, + aws_keys.first().unwrap().0, AmazonS3ConfigKey::SecretAccessKey ); assert_eq!(aws_keys.len(), 1); From 48c55e0439c0d4c1e87b64eb975955eb81abdd20 Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Sun, 24 Nov 2024 14:34:17 +0100 Subject: [PATCH 5/6] chore: add comment --- crates/polars-io/src/cloud/options.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/polars-io/src/cloud/options.rs b/crates/polars-io/src/cloud/options.rs index dd800185ce53..93d92c92265f 100644 --- a/crates/polars-io/src/cloud/options.rs +++ b/crates/polars-io/src/cloud/options.rs @@ -122,6 +122,7 @@ where { Ok(config .into_iter() + // Silenty ignores custom upstream storage_options .filter_map(|(key, val)| { T::from_str(key.as_ref().to_ascii_lowercase().as_str()) .ok() From 6ec5f322a45acb8fed7402005e62d1a210350b9d Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Sun, 24 Nov 2024 14:38:40 +0100 Subject: [PATCH 6/6] typo --- crates/polars-io/src/cloud/options.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/polars-io/src/cloud/options.rs b/crates/polars-io/src/cloud/options.rs index 93d92c92265f..545ab5550890 100644 --- a/crates/polars-io/src/cloud/options.rs +++ b/crates/polars-io/src/cloud/options.rs @@ -122,7 +122,7 @@ where { Ok(config .into_iter() - // Silenty ignores custom upstream storage_options + // Silently ignores custom upstream storage_options .filter_map(|(key, val)| { T::from_str(key.as_ref().to_ascii_lowercase().as_str()) .ok()