Merge remote-tracking branch 'apache/main' into feature/13753-udlp-un…

…parser

Cargo.toml

@@ -66,14 +66,14 @@ license = "Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/apache/datafusion"
 rust-version = "1.80.1"
-version = "43.0.0"
+version = "44.0.0"
 
 [workspace.dependencies]
 # We turn off default-features for some dependencies here so the workspaces which inherit them can
 # selectively turn them on if needed, since we can override default-features = true (from false)
 # for the inherited dependency but cannot do the reverse (override from true to false).
 #
-# See for more detaiils: https://github.com/rust-lang/cargo/issues/11329
+# See for more details: https://github.com/rust-lang/cargo/issues/11329
 ahash = { version = "0.8", default-features = false, features = [
     "runtime-rng",
 ] }
@@ -98,31 +98,31 @@ bytes = "1.4"
 chrono = { version = "0.4.38", default-features = false }
 ctor = "0.2.0"
 dashmap = "6.0.1"
-datafusion = { path = "datafusion/core", version = "43.0.0", default-features = false }
-datafusion-catalog = { path = "datafusion/catalog", version = "43.0.0" }
-datafusion-common = { path = "datafusion/common", version = "43.0.0", default-features = false }
-datafusion-common-runtime = { path = "datafusion/common-runtime", version = "43.0.0" }
-datafusion-doc = { path = "datafusion/doc", version = "43.0.0" }
-datafusion-execution = { path = "datafusion/execution", version = "43.0.0" }
-datafusion-expr = { path = "datafusion/expr", version = "43.0.0" }
-datafusion-expr-common = { path = "datafusion/expr-common", version = "43.0.0" }
-datafusion-ffi = { path = "datafusion/ffi", version = "43.0.0" }
-datafusion-functions = { path = "datafusion/functions", version = "43.0.0" }
-datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "43.0.0" }
-datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "43.0.0" }
-datafusion-functions-nested = { path = "datafusion/functions-nested", version = "43.0.0" }
-datafusion-functions-table = { path = "datafusion/functions-table", version = "43.0.0" }
-datafusion-functions-window = { path = "datafusion/functions-window", version = "43.0.0" }
-datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "43.0.0" }
-datafusion-macros = { path = "datafusion/macros", version = "43.0.0" }
-datafusion-optimizer = { path = "datafusion/optimizer", version = "43.0.0", default-features = false }
-datafusion-physical-expr = { path = "datafusion/physical-expr", version = "43.0.0", default-features = false }
-datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "43.0.0", default-features = false }
-datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "43.0.0" }
-datafusion-physical-plan = { path = "datafusion/physical-plan", version = "43.0.0" }
-datafusion-proto = { path = "datafusion/proto", version = "43.0.0" }
-datafusion-proto-common = { path = "datafusion/proto-common", version = "43.0.0" }
-datafusion-sql = { path = "datafusion/sql", version = "43.0.0" }
+datafusion = { path = "datafusion/core", version = "44.0.0", default-features = false }
+datafusion-catalog = { path = "datafusion/catalog", version = "44.0.0" }
+datafusion-common = { path = "datafusion/common", version = "44.0.0", default-features = false }
+datafusion-common-runtime = { path = "datafusion/common-runtime", version = "44.0.0" }
+datafusion-doc = { path = "datafusion/doc", version = "44.0.0" }
+datafusion-execution = { path = "datafusion/execution", version = "44.0.0" }
+datafusion-expr = { path = "datafusion/expr", version = "44.0.0" }
+datafusion-expr-common = { path = "datafusion/expr-common", version = "44.0.0" }
+datafusion-ffi = { path = "datafusion/ffi", version = "44.0.0" }
+datafusion-functions = { path = "datafusion/functions", version = "44.0.0" }
+datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "44.0.0" }
+datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "44.0.0" }
+datafusion-functions-nested = { path = "datafusion/functions-nested", version = "44.0.0" }
+datafusion-functions-table = { path = "datafusion/functions-table", version = "44.0.0" }
+datafusion-functions-window = { path = "datafusion/functions-window", version = "44.0.0" }
+datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "44.0.0" }
+datafusion-macros = { path = "datafusion/macros", version = "44.0.0" }
+datafusion-optimizer = { path = "datafusion/optimizer", version = "44.0.0", default-features = false }
+datafusion-physical-expr = { path = "datafusion/physical-expr", version = "44.0.0", default-features = false }
+datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "44.0.0", default-features = false }
+datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "44.0.0" }
+datafusion-physical-plan = { path = "datafusion/physical-plan", version = "44.0.0" }
+datafusion-proto = { path = "datafusion/proto", version = "44.0.0" }
+datafusion-proto-common = { path = "datafusion/proto-common", version = "44.0.0" }
+datafusion-sql = { path = "datafusion/sql", version = "44.0.0" }
 doc-comment = "0.3"
 env_logger = "0.11"
 futures = "0.3"

README.md

@@ -112,7 +112,8 @@ Default features:
 - `parquet`: support for reading the [Apache Parquet] format
 - `regex_expressions`: regular expression functions, such as `regexp_match`
 - `unicode_expressions`: Include unicode aware functions such as `character_length`
-- `unparser` : enables support to reverse LogicalPlans back into SQL
+- `unparser`: enables support to reverse LogicalPlans back into SQL
+- `recursive_protection`: uses [recursive](https://docs.rs/recursive/latest/recursive/) for stack overflow protection.
 
 Optional features:
 

datafusion-cli/Cargo.toml

@@ -18,7 +18,7 @@
 [package]
 name = "datafusion-cli"
 description = "Command Line Client for DataFusion query engine."
-version = "43.0.0"
+version = "44.0.0"
 authors = ["Apache DataFusion <[email protected]>"]
 edition = "2021"
 keywords = ["arrow", "datafusion", "query", "sql"]
@@ -39,17 +39,18 @@ aws-sdk-sts = "1.43.0"
 # end pin aws-sdk crates
 aws-credential-types = "1.2.0"
 clap = { version = "4.5.16", features = ["derive", "cargo"] }
-datafusion = { path = "../datafusion/core", version = "43.0.0", features = [
+datafusion = { path = "../datafusion/core", version = "44.0.0", features = [
     "avro",
     "crypto_expressions",
     "datetime_expressions",
     "encoding_expressions",
     "parquet",
+    "recursive_protection",
     "regex_expressions",
     "unicode_expressions",
     "compression",
 ] }
-datafusion-catalog = { path = "../datafusion/catalog", version = "43.0.0" }
+datafusion-catalog = { path = "../datafusion/catalog", version = "44.0.0" }
 dirs = "5.0.1"
 env_logger = "0.11"
 futures = "0.3"

datafusion-cli/src/functions.rs

@@ -360,7 +360,7 @@ impl TableFunctionImpl for ParquetMetadataFunc {
             Field::new("total_uncompressed_size", DataType::Int64, true),
         ]));
 
-        // construct recordbatch from metadata
+        // construct record batch from metadata
         let mut filename_arr = vec![];
         let mut row_group_id_arr = vec![];
         let mut row_group_num_rows_arr = vec![];

datafusion-examples/README.md

@@ -22,7 +22,7 @@
 This crate includes end to end, highly commented examples of how to use
 various DataFusion APIs to help you get started.
 
-## Prerequisites:
+## Prerequisites
 
 Run `git submodule update --init` to init test files.
 
@@ -57,8 +57,7 @@ cargo run --example dataframe
 - [`custom_datasource.rs`](examples/custom_datasource.rs): Run queries against a custom datasource (TableProvider)
 - [`custom_file_format.rs`](examples/custom_file_format.rs): Write data to a custom file format
 - [`dataframe-to-s3.rs`](examples/external_dependency/dataframe-to-s3.rs): Run a query using a DataFrame against a parquet file from s3 and writing back to s3
-- [`dataframe.rs`](examples/dataframe.rs): Run a query using a DataFrame API against parquet files, csv files, and in-memory data
-- [`dataframe_output.rs`](examples/dataframe_output.rs): Examples of methods which write data out from a DataFrame
+- [`dataframe.rs`](examples/dataframe.rs): Run a query using a DataFrame API against parquet files, csv files, and in-memory data. Also demonstrates the various methods to write out a DataFrame to a table, parquet file, csv file, and json file.
 - [`deserialize_to_struct.rs`](examples/deserialize_to_struct.rs): Convert query results into rust structs using serde
 - [`expr_api.rs`](examples/expr_api.rs): Create, execute, simplify, analyze and coerce `Expr`s
 - [`file_stream_provider.rs`](examples/file_stream_provider.rs): Run a query on `FileStreamProvider` which implements `StreamProvider` for reading and writing to arbitrary stream sources / sinks.

datafusion-examples/examples/advanced_parquet_index.rs

@@ -82,7 +82,7 @@ use url::Url;
 /// Specifically, this example illustrates how to:
 /// 1. Use [`ParquetFileReaderFactory`] to avoid re-reading parquet metadata on each query
 /// 2. Use [`PruningPredicate`] for predicate analysis
-/// 3. Pass a row group selection to [`ParuetExec`]
+/// 3. Pass a row group selection to [`ParquetExec`]
 /// 4. Pass a row selection (within a row group) to [`ParquetExec`]
 ///
 /// Note this is a *VERY* low level example for people who want to build their
@@ -211,7 +211,7 @@ async fn main() -> Result<()> {
     //
     // Note: in order to prune pages, the Page Index must be loaded and the
     // ParquetExec will load it on demand if not present. To avoid a second IO
-    // during query, this example loaded the Page Index pre-emptively by setting
+    // during query, this example loaded the Page Index preemptively by setting
     // `ArrowReader::with_page_index` in `IndexedFile::try_new`
     provider.set_use_row_selection(true);
     println!("** Select data, predicate `id = 950`");