From c2aab931e233e519431a58c395365e57a4613f7d Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Thu, 8 Aug 2024 15:19:17 +0200 Subject: [PATCH] Remove many `crate::` imports in listing table provider module This is part of isolating this module in order to be able to move it out of core. This commit attempts to replace all `crate::` imports that are possible to avoid today (i.e. which would be replaced when listing table provider was moved to separate crate), leaving those that cannot be replaced. This makes it easy to notice the remaining coupling between the listing table provider module and the core. --- .../core/src/datasource/listing/helpers.rs | 8 +++--- datafusion/core/src/datasource/listing/mod.rs | 4 +-- .../core/src/datasource/listing/table.rs | 25 ++++++++----------- datafusion/core/src/datasource/listing/url.rs | 2 +- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/datafusion/core/src/datasource/listing/helpers.rs b/datafusion/core/src/datasource/listing/helpers.rs index 67af8ef12c8b..b5dd2dd12e10 100644 --- a/datafusion/core/src/datasource/listing/helpers.rs +++ b/datafusion/core/src/datasource/listing/helpers.rs @@ -21,11 +21,11 @@ use std::collections::HashMap; use std::mem; use std::sync::Arc; +use super::ListingTableUrl; use super::PartitionedFile; -use crate::datasource::listing::ListingTableUrl; use crate::execution::context::SessionState; -use crate::logical_expr::{BinaryExpr, Operator}; -use crate::{error::Result, scalar::ScalarValue}; +use datafusion_common::{Result, ScalarValue}; +use datafusion_expr::{BinaryExpr, Operator}; use arrow::{ array::{Array, ArrayRef, AsArray, StringBuilder}, @@ -518,8 +518,8 @@ mod tests { use futures::StreamExt; - use crate::logical_expr::{case, col, lit, Expr}; use crate::test::object_store::make_test_store_and_state; + use datafusion_expr::{case, col, lit, Expr}; use super::*; diff --git a/datafusion/core/src/datasource/listing/mod.rs b/datafusion/core/src/datasource/listing/mod.rs index 21a60614cff2..c5a441aacf1d 100644 --- a/datafusion/core/src/datasource/listing/mod.rs +++ b/datafusion/core/src/datasource/listing/mod.rs @@ -22,8 +22,8 @@ mod helpers; mod table; mod url; -use crate::error::Result; use chrono::TimeZone; +use datafusion_common::Result; use datafusion_common::{ScalarValue, Statistics}; use futures::Stream; use object_store::{path::Path, ObjectMeta}; @@ -162,7 +162,7 @@ impl From for PartitionedFile { #[cfg(test)] mod tests { - use crate::datasource::listing::ListingTableUrl; + use super::ListingTableUrl; use datafusion_execution::object_store::{ DefaultObjectStoreRegistry, ObjectStoreRegistry, }; diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index bb86ac3ae416..89066d8234ac 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -24,19 +24,18 @@ use std::{any::Any, sync::Arc}; use super::helpers::{expr_applicable_for_cols, pruned_partition_list, split_files}; use super::PartitionedFile; -use crate::catalog::TableProvider; -use crate::datasource::{create_ordering, get_statistics_with_limit, TableType}; +use super::ListingTableUrl; +use crate::datasource::{create_ordering, get_statistics_with_limit}; use crate::datasource::{ file_format::{file_compression_type::FileCompressionType, FileFormat}, - listing::ListingTableUrl, physical_plan::{FileScanConfig, FileSinkConfig}, }; -use crate::{ - error::{DataFusionError, Result}, - execution::context::SessionState, - logical_expr::{utils::conjunction, Expr, TableProviderFilterPushDown}, - physical_plan::{empty::EmptyExec, ExecutionPlan, Statistics}, -}; +use crate::execution::context::SessionState; +use datafusion_catalog::TableProvider; +use datafusion_common::{DataFusionError, Result}; +use datafusion_expr::TableType; +use datafusion_expr::{utils::conjunction, Expr, TableProviderFilterPushDown}; +use datafusion_physical_plan::{empty::EmptyExec, ExecutionPlan, Statistics}; use arrow::datatypes::{DataType, Field, SchemaBuilder, SchemaRef}; use arrow_schema::Schema; @@ -1051,12 +1050,12 @@ mod tests { use crate::datasource::file_format::parquet::ParquetFormat; use crate::datasource::{provider_as_source, MemTable}; use crate::execution::options::ArrowReadOptions; - use crate::physical_plan::collect; use crate::prelude::*; use crate::{ assert_batches_eq, test::{columns, object_store::register_test_store}, }; + use datafusion_physical_plan::collect; use arrow::record_batch::RecordBatch; use arrow_schema::SortOptions; @@ -1154,10 +1153,8 @@ mod tests { let options = ListingOptions::new(Arc::new(ParquetFormat::default())); let schema = options.infer_schema(&state, &table_path).await.unwrap(); - use crate::{ - datasource::file_format::parquet::ParquetFormat, - physical_plan::expressions::col as physical_col, - }; + use crate::datasource::file_format::parquet::ParquetFormat; + use datafusion_physical_plan::expressions::col as physical_col; use std::ops::Add; // (file_sort_order, expected_result) diff --git a/datafusion/core/src/datasource/listing/url.rs b/datafusion/core/src/datasource/listing/url.rs index 7566df628ed7..1701707fdb72 100644 --- a/datafusion/core/src/datasource/listing/url.rs +++ b/datafusion/core/src/datasource/listing/url.rs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -use crate::datasource::object_store::ObjectStoreUrl; use crate::execution::context::SessionState; use datafusion_common::{DataFusionError, Result}; +use datafusion_execution::object_store::ObjectStoreUrl; use datafusion_optimizer::OptimizerConfig; use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt};