Merge branch 'main' into case_improvements

Author: alamb

.asf.yaml

@@ -41,6 +41,7 @@ github:
     - sql
   enabled_merge_buttons:
     squash: true
+    squash_commit_message: PR_TITLE_AND_DESC
     merge: false
     rebase: false
   features:

.github/workflows/audit.yml

@@ -42,7 +42,7 @@ jobs:
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
       - name: Install cargo-audit
-        uses: taiki-e/install-action@e7ef886cf8f69c25ecef6bbc2858a42e273496ec  # v2.62.28
+        uses: taiki-e/install-action@e43a5023a747770bfcb71ae048541a681714b951  # v2.62.33
         with:
           tool: cargo-audit
       - name: Run audit check

.github/workflows/extended.yml

@@ -36,6 +36,14 @@ on:
       # it is not expected to have many changes in these branches,
       # so running extended tests is not a burden
       - 'branch-*'
+  # Also run for changes to some critical areas that are most likely
+  # to trigger errors in extended tests
+  pull_request:
+    branches: [ '**' ]
+    paths:
+      - 'datafusion/physical*/**/*.rs'
+      - 'datafusion/expr*/**/*.rs'
+      - 'datafusion/optimizer/**/*.rs'
   workflow_dispatch:
     inputs:
       pr_number:

.github/workflows/labeler/labeler-config.yml

@@ -58,7 +58,7 @@ execution:
 
 datasource:
   - changed-files:
-      - any-glob-to-any-file: ['datafusion/datasource/**/*', 'datafusion/datasource-avro/**/*', 'datafusion/datasource-csv/**/*', 'datafusion/datasource-json/**/*', 'datafusion/datasource-parquet/**/*']
+      - any-glob-to-any-file: ['datafusion/datasource/**/*', 'datafusion/datasource-avro/**/*', 'datafusion/datasource-arrow/**/*', 'datafusion/datasource-csv/**/*', 'datafusion/datasource-json/**/*', 'datafusion/datasource-parquet/**/*']
 
 functions:
   - changed-files:

.github/workflows/rust.yml

@@ -412,7 +412,7 @@ jobs:
           sudo apt-get update -qq
           sudo apt-get install -y -qq clang
       - name: Setup wasm-pack
-        uses: taiki-e/install-action@e7ef886cf8f69c25ecef6bbc2858a42e273496ec  # v2.62.28
+        uses: taiki-e/install-action@e43a5023a747770bfcb71ae048541a681714b951  # v2.62.33
         with:
           tool: wasm-pack
       - name: Run tests with headless mode
@@ -739,7 +739,7 @@ jobs:
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
       - name: Install cargo-msrv
-        uses: taiki-e/install-action@e7ef886cf8f69c25ecef6bbc2858a42e273496ec  # v2.62.28
+        uses: taiki-e/install-action@e43a5023a747770bfcb71ae048541a681714b951  # v2.62.33
         with:
           tool: cargo-msrv
 

Cargo.lock

@@ -22,6 +22,7 @@ members = [
     "datafusion/catalog",
     "datafusion/catalog-listing",
     "datafusion/datasource",
+    "datafusion/datasource-arrow",
     "datafusion/datasource-avro",
     "datafusion/datasource-csv",
     "datafusion/datasource-json",
@@ -116,6 +117,7 @@ datafusion-catalog-listing = { path = "datafusion/catalog-listing", version = "5
 datafusion-common = { path = "datafusion/common", version = "50.2.0", default-features = false }
 datafusion-common-runtime = { path = "datafusion/common-runtime", version = "50.2.0" }
 datafusion-datasource = { path = "datafusion/datasource", version = "50.2.0", default-features = false }
+datafusion-datasource-arrow = { path = "datafusion/datasource-arrow", version = "50.2.0", default-features = false }
 datafusion-datasource-avro = { path = "datafusion/datasource-avro", version = "50.2.0", default-features = false }
 datafusion-datasource-csv = { path = "datafusion/datasource-csv", version = "50.2.0", default-features = false }
 datafusion-datasource-json = { path = "datafusion/datasource-json", version = "50.2.0", default-features = false }

Cargo.toml

@@ -40,7 +40,7 @@ async-trait = { workspace = true }
 aws-config = "1.8.7"
 aws-credential-types = "1.2.7"
 chrono = { workspace = true }
-clap = { version = "4.5.47", features = ["derive", "cargo"] }
+clap = { version = "4.5.47", features = ["cargo", "derive"] }
 datafusion = { workspace = true, features = [
     "avro",
     "compression",
@@ -55,6 +55,7 @@ datafusion = { workspace = true, features = [
     "sql",
     "unicode_expressions",
 ] }
+datafusion-common = { workspace = true }
 dirs = "6.0.0"
 env_logger = { workspace = true }
 futures = { workspace = true }
@@ -65,7 +66,7 @@ parking_lot = { workspace = true }
 parquet = { workspace = true, default-features = false }
 regex = { workspace = true }
 rustyline = "17.0"
-tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot", "signal"] }
+tokio = { workspace = true, features = ["macros", "parking_lot", "rt", "rt-multi-thread", "signal", "sync"] }
 url = { workspace = true }
 
 [dev-dependencies]

datafusion-cli/Cargo.toml

@@ -128,7 +128,7 @@ impl Command {
                     let profile_mode = mode
                         .parse()
                         .map_err(|_|
-                            exec_datafusion_err!("Failed to parse input: {mode}. Valid options are disabled, enabled")
+                            exec_datafusion_err!("Failed to parse input: {mode}. Valid options are disabled, summary, trace")
                         )?;
                     print_options
                         .instrumented_registry
@@ -165,7 +165,7 @@ impl Command {
                 ("\\pset [NAME [VALUE]]", "set table output option\n(format)")
             }
             Self::ObjectStoreProfileMode(_) => (
-                "\\object_store_profiling (disabled|enabled)",
+                "\\object_store_profiling (disabled|summary|trace)",
                 "print or set object store profile mode",
             ),
         }
@@ -312,13 +312,22 @@ mod tests {
             InstrumentedObjectStoreMode::default()
         );
 
-        cmd = "object_store_profiling enabled"
+        cmd = "object_store_profiling summary"
             .parse()
             .expect("expected parse to succeed");
         assert!(cmd.execute(&ctx, &mut print_options).await.is_ok());
         assert_eq!(
             print_options.instrumented_registry.instrument_mode(),
-            InstrumentedObjectStoreMode::Enabled
+            InstrumentedObjectStoreMode::Summary
+        );
+
+        cmd = "object_store_profiling trace"
+            .parse()
+            .expect("expected parse to succeed");
+        assert!(cmd.execute(&ctx, &mut print_options).await.is_ok());
+        assert_eq!(
+            print_options.instrumented_registry.instrument_mode(),
+            InstrumentedObjectStoreMode::Trace
         );
 
         cmd = "object_store_profiling does_not_exist"

datafusion-cli/src/command.rs

@@ -24,6 +24,7 @@ use crate::highlighter::{NoSyntaxHighlighter, SyntaxHighlighter};
 
 use datafusion::sql::parser::{DFParser, Statement};
 use datafusion::sql::sqlparser::dialect::dialect_from_str;
+use datafusion_common::config::Dialect;
 
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::error::ReadlineError;
@@ -34,33 +35,33 @@ use rustyline::{Context, Helper, Result};
 
 pub struct CliHelper {
     completer: FilenameCompleter,
-    dialect: String,
+    dialect: Dialect,
     highlighter: Box<dyn Highlighter>,
 }
 
 impl CliHelper {
-    pub fn new(dialect: &str, color: bool) -> Self {
+    pub fn new(dialect: &Dialect, color: bool) -> Self {
         let highlighter: Box<dyn Highlighter> = if !color {
             Box::new(NoSyntaxHighlighter {})
         } else {
             Box::new(SyntaxHighlighter::new(dialect))
         };
         Self {
             completer: FilenameCompleter::new(),
-            dialect: dialect.into(),
+            dialect: *dialect,
             highlighter,
         }
     }
 
-    pub fn set_dialect(&mut self, dialect: &str) {
-        if dialect != self.dialect {
-            self.dialect = dialect.to_string();
+    pub fn set_dialect(&mut self, dialect: &Dialect) {
+        if *dialect != self.dialect {
+            self.dialect = *dialect;
         }
     }
 
     fn validate_input(&self, input: &str) -> Result<ValidationResult> {
         if let Some(sql) = input.strip_suffix(';') {
-            let dialect = match dialect_from_str(&self.dialect) {
+            let dialect = match dialect_from_str(self.dialect) {
                 Some(dialect) => dialect,
                 None => {
                     return Ok(ValidationResult::Invalid(Some(format!(
@@ -97,7 +98,7 @@ impl CliHelper {
 
 impl Default for CliHelper {
     fn default() -> Self {
-        Self::new("generic", false)
+        Self::new(&Dialect::Generic, false)
     }
 }
 
@@ -289,7 +290,7 @@ mod tests {
         );
 
         // valid in postgresql dialect
-        validator.set_dialect("postgresql");
+        validator.set_dialect(&Dialect::PostgreSQL);
         let result =
             readline_direct(Cursor::new(r"select 1 # 2;".as_bytes()), &validator)?;
         assert!(matches!(result, ValidationResult::Valid(None)));