Merge branch 'apache:main' into feature-scalar_regexp_match_expr

.github/workflows/rust.yml

@@ -80,9 +80,12 @@ jobs:
       - name: Check datafusion-common without default features
         run: cargo check --all-targets --no-default-features -p datafusion-common
 
-      - name: Check datafusion-functions
+      - name: Check datafusion-functions without default features
         run: cargo check --all-targets --no-default-features -p datafusion-functions
 
+      - name: Check datafusion-substrait without default features
+        run: cargo check --all-targets --no-default-features -p datafusion-substrait
+
       - name: Check workspace in debug mode
         run: cargo check --all-targets --workspace
 
@@ -582,9 +585,9 @@ jobs:
           #
           # To reproduce: 
           # 1. Install the version of Rust that is failing. Example: 
-          #    rustup install 1.79.0
+          #    rustup install 1.80.0
           # 2. Run the command that failed with that version. Example:
-          #    cargo +1.79.0 check -p datafusion
+          #    cargo +1.80.0 check -p datafusion
           # 
           # To resolve, either:
           # 1. Change your code to use older Rust features, 
@@ -603,4 +606,4 @@ jobs:
         run: cargo msrv --output-format json --log-target stdout verify
       - name: Check datafusion-cli
         working-directory: datafusion-cli
-        run: cargo msrv --output-format json --log-target stdout verify
+        run: cargo msrv --output-format json --log-target stdout verify

Cargo.toml

@@ -65,7 +65,7 @@ homepage = "https://datafusion.apache.org"
 license = "Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/apache/datafusion"
-rust-version = "1.79"
+rust-version = "1.80"
 version = "43.0.0"
 
 [workspace.dependencies]

datafusion-cli/Cargo.toml

@@ -26,7 +26,7 @@ license = "Apache-2.0"
 homepage = "https://datafusion.apache.org"
 repository = "https://github.com/apache/datafusion"
 # Specify MSRV here as `cargo msrv` doesn't support workspace version
-rust-version = "1.79"
+rust-version = "1.80"
 readme = "README.md"
 
 [dependencies]

datafusion-cli/Dockerfile

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-FROM rust:1.79-bookworm AS builder
+FROM rust:1.80-bookworm AS builder
 
 COPY . /usr/src/datafusion
 COPY ./datafusion /usr/src/datafusion/datafusion

datafusion-cli/src/object_storage.rs

@@ -32,7 +32,7 @@ use aws_credential_types::provider::ProvideCredentials;
 use object_store::aws::{AmazonS3Builder, AwsCredential};
 use object_store::gcp::GoogleCloudStorageBuilder;
 use object_store::http::HttpBuilder;
-use object_store::{CredentialProvider, ObjectStore};
+use object_store::{ClientOptions, CredentialProvider, ObjectStore};
 use url::Url;
 
 pub async fn get_s3_object_store_builder(
@@ -437,6 +437,7 @@ pub(crate) async fn get_object_store(
         }
         "http" | "https" => Arc::new(
             HttpBuilder::new()
+                .with_client_options(ClientOptions::new().with_allow_http(true))
                 .with_url(url.origin().ascii_serialization())
                 .build()?,
         ),

datafusion/common/src/cse.rs

@@ -24,8 +24,8 @@ use crate::tree_node::{
     Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter,
     TreeNodeVisitor,
 };
+use crate::IndexMap;
 use crate::Result;
-use indexmap::IndexMap;
 use std::collections::HashMap;
 use std::hash::{BuildHasher, Hash, Hasher, RandomState};
 use std::marker::PhantomData;

datafusion/common/src/dfschema.rs

@@ -1382,14 +1382,14 @@ mod tests {
 
         // Succeeds if both have the same element type, disregards names and nullability
         assert!(DFSchema::datatype_is_logically_equal(
-            &DataType::List(Field::new("item", DataType::Int8, true).into()),
+            &DataType::List(Field::new_list_field(DataType::Int8, true).into()),
             &DataType::List(Field::new("element", DataType::Int8, false).into())
         ));
 
         // Fails if element type is different
         assert!(!DFSchema::datatype_is_logically_equal(
-            &DataType::List(Field::new("item", DataType::Int8, true).into()),
-            &DataType::List(Field::new("item", DataType::Int16, true).into())
+            &DataType::List(Field::new_list_field(DataType::Int8, true).into()),
+            &DataType::List(Field::new_list_field(DataType::Int16, true).into())
         ));
 
         // Test maps
@@ -1522,14 +1522,14 @@ mod tests {
 
         // Succeeds if both have the same element type, disregards names and nullability
         assert!(DFSchema::datatype_is_semantically_equal(
-            &DataType::List(Field::new("item", DataType::Int8, true).into()),
+            &DataType::List(Field::new_list_field(DataType::Int8, true).into()),
             &DataType::List(Field::new("element", DataType::Int8, false).into())
         ));
 
         // Fails if element type is different
         assert!(!DFSchema::datatype_is_semantically_equal(
-            &DataType::List(Field::new("item", DataType::Int8, true).into()),
-            &DataType::List(Field::new("item", DataType::Int16, true).into())
+            &DataType::List(Field::new_list_field(DataType::Int8, true).into()),
+            &DataType::List(Field::new_list_field(DataType::Int16, true).into())
         ));
 
         // Test maps

datafusion/common/src/lib.rs

@@ -73,6 +73,7 @@ pub use param_value::ParamValues;
 pub use scalar::{ScalarType, ScalarValue};
 pub use schema_reference::SchemaReference;
 pub use stats::{ColumnStatistics, Statistics};
+use std::hash::RandomState;
 pub use table_reference::{ResolvedTableReference, TableReference};
 pub use unnest::{RecursionUnnestOption, UnnestOptions};
 pub use utils::project_schema;
@@ -93,6 +94,9 @@ pub use error::{
 pub type HashMap<K, V, S = DefaultHashBuilder> = hashbrown::HashMap<K, V, S>;
 pub type HashSet<T, S = DefaultHashBuilder> = hashbrown::HashSet<T, S>;
 
+pub type IndexMap<T, S = DefaultHashBuilder> = indexmap::IndexMap<T, S>;
+pub type IndexSet<T, S = RandomState> = indexmap::IndexSet<T, S>;
+
 /// Downcast an Arrow Array to a concrete type, return an `DataFusionError::Internal` if the cast is
 /// not possible. In normal usage of DataFusion the downcast should always succeed.
 ///