Remove expression.rs file and use Arrow DataFusion Python version

.github/CODEOWNERS

@@ -2,4 +2,7 @@
 *              @ayushdg @charlesbluca @galipremsagar
 
 # rust codeowners
-dask_planner/  @ayushdg @charlesbluca @galipremsagar @jdye64
+.cargo/        @ayushdg @charlesbluca @galipremsagar @jdye64
+src/           @ayushdg @charlesbluca @galipremsagar @jdye64
+Cargo.toml     @ayushdg @charlesbluca @galipremsagar @jdye64
+Cargo.lock     @ayushdg @charlesbluca @galipremsagar @jdye64

.github/workflows/conda.yml

@@ -6,10 +6,9 @@ on:
   pull_request:
     paths:
       - setup.py
-      - dask_planner/Cargo.toml
-      - dask_planner/Cargo.lock
-      - dask_planner/pyproject.toml
-      - dask_planner/rust-toolchain.toml
+      - Cargo.toml
+      - Cargo.lock
+      - pyproject.toml
       - continuous_integration/recipe/**
       - .github/workflows/conda.yml
   schedule:

.github/workflows/release.yml

@@ -60,21 +60,21 @@ jobs:
             CARGO_NET_GIT_FETCH_WITH_CLI="true"
             PATH="$HOME/.cargo/bin:$HOME/.local/bin:$PATH"
           CIBW_ENVIRONMENT_WINDOWS: 'PATH="$UserProfile\.cargo\bin;$PATH"'
-          CIBW_BEFORE_BUILD: 'pip install -U setuptools-rust'
+          CIBW_BEFORE_BUILD: 'pip install -U "maturin>=0.15,<0.16"'
           CIBW_BEFORE_BUILD_LINUX: >
             ARCH=$([ $(uname -m) == x86_64 ] && echo x86_64 || echo aarch_64) &&
             DOWNLOAD_URL=$(curl --retry 6 --retry-delay 10 -s https://api.github.com/repos/protocolbuffers/protobuf/releases/latest | grep -o '"browser_download_url": "[^"]*' | cut -d'"' -f4 | grep "\linux-${ARCH}.zip$") &&
             curl --retry 6 --retry-delay 10 -LO $DOWNLOAD_URL &&
             unzip protoc-*-linux-$ARCH.zip -d $HOME/.local &&
             protoc --version &&
-            pip install -U setuptools-rust &&
+            pip install -U "maturin>=0.15,<0.16" &&
             pip list &&
             curl --retry 6 --retry-delay 10 https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=stable --profile=minimal -y &&
             rustup show
         with:
           package-dir: .
           output-dir: dist
-          config-file: "dask_planner/pyproject.toml"
+          config-file: "pyproject.toml"
       - name: Set up Python
         uses: conda-incubator/[email protected]
         with:
@@ -127,7 +127,7 @@ jobs:
           channel-priority: strict
       - name: Build source distribution
         run: |
-          mamba install setuptools-rust twine
+          mamba install "maturin>=0.15,<0.16" twine
 
           python setup.py sdist
       - name: Check dist files

.github/workflows/rust.yml

@@ -51,7 +51,6 @@ jobs:
       - name: Optionally update upstream dependencies
         if: needs.detect-ci-trigger.outputs.triggered == 'true'
         run: |
-          cd dask_planner
           bash update-dependencies.sh
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
@@ -60,11 +59,9 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Check workspace in debug mode
         run: |
-          cd dask_planner
           cargo check
       - name: Check workspace in release mode
         run: |
-          cd dask_planner
           cargo check --release
 
   # test the crate
@@ -84,7 +81,6 @@ jobs:
       - name: Optionally update upstream dependencies
         if: needs.detect-ci-trigger.outputs.triggered == 'true'
         run: |
-          cd dask_planner
           bash update-dependencies.sh
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
@@ -93,5 +89,4 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Run tests
         run: |
-          cd dask_planner
           cargo test

.github/workflows/test-upstream.yml

@@ -68,11 +68,10 @@ jobs:
       - name: Optionally update upstream cargo dependencies
         if: env.which_upstream == 'DataFusion'
         run: |
-          cd dask_planner
           bash update-dependencies.sh
       - name: Build the Rust DataFusion bindings
         run: |
-          python setup.py build install
+          maturin develop
       - name: Install hive testing dependencies
         if: matrix.os == 'ubuntu-latest'
         run: |
@@ -122,11 +121,9 @@ jobs:
         env:
           UPDATE_ALL_CARGO_DEPS: false
         run: |
-          cd dask_planner
           bash update-dependencies.sh
       - name: Install dependencies and nothing else
         run: |
-          mamba install setuptools-rust
           pip install -e . -vv
 
           which python

.github/workflows/test.yml

@@ -72,7 +72,7 @@ jobs:
           shared-key: test
       - name: Build the Rust DataFusion bindings
         run: |
-          python setup.py build install
+          maturin develop
       - name: Install hive testing dependencies
         if: matrix.os == 'ubuntu-latest'
         run: |
@@ -116,7 +116,6 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Install dependencies and nothing else
         run: |
-          mamba install "setuptools-rust>=1.5.2"
           pip install -e . -vv
 
           which python

.gitignore

@@ -46,23 +46,15 @@ venv
 # IDE
 .idea
 .vscode
-planner/.classpath
-planner/.project
-planner/.settings/
-planner/.idea
-planner/*.iml
 *.swp
 
 # project specific
-planner/dependency-reduced-pom.xml
-planner/target/
-dask_sql/jar
-.next/
 dask-worker-space/
 node_modules/
 docs/source/_build/
 tests/unit/queries
 tests/unit/data
+target/*
 
 # Ignore development specific local testing files
 dev_tests

.pre-commit-config.yaml

@@ -20,9 +20,9 @@ repos:
     rev: v1.0
     hooks:
       - id: cargo-check
-        args: ['--manifest-path', './dask_planner/Cargo.toml', '--verbose', '--']
+        args: ['--manifest-path', './Cargo.toml', '--verbose', '--']
       - id: clippy
-        args: ['--manifest-path', './dask_planner/Cargo.toml', '--verbose', '--', '-D', 'warnings']
+        args: ['--manifest-path', './Cargo.toml', '--verbose', '--', '-D', 'warnings']
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.2.0
     hooks:
@@ -39,4 +39,4 @@ repos:
         entry: cargo +nightly fmt
         language: system
         types: [rust]
-        args: ['--manifest-path', './dask_planner/Cargo.toml', '--verbose', '--']
+        args: ['--manifest-path', './Cargo.toml', '--verbose', '--']

CONTRIBUTING.md

@@ -49,15 +49,15 @@ Note that while `setuptools-rust` is used by CI and should be used during your d
 Building Dask-SQL is straightforward with Python. To build run ```python setup.py install```. This will build both the Rust and Python codebase and install it into your locally activated conda environment. While not required, if you have updated dependencies for Rust you might prefer a clean build. To clean your setup run ```python setup.py clean``` and then run ```python setup.py install```
 
 #### DataFusion Modules
-DataFusion is broken down into a few modules. We consume those modules in our [Cargo.toml](dask_planner/Cargo.toml). The modules that we use currently are
+DataFusion is broken down into a few modules. We consume those modules in our [Cargo.toml](Cargo.toml). The modules that we use currently are
 
 - `datafusion-common` - Datastructures and core logic
 - `datafusion-expr` - Expression based logic and operators
 - `datafusion-sql` - SQL components such as parsing and planning
 - `datafusion-optimizer` - Optimization logic and datastructures for modifying current plans into more efficient ones.
 
 #### Retrieving Upstream Dependencies
-During development you might find yourself needing some upstream DataFusion changes not present in the projects current version. Luckily this can easily be achieved by updating [Cargo.toml](dask_planner/Cargo.toml) and changing the `rev` to the SHA of the version you need. Note that the same SHA should be used for all DataFusion modules.
+During development you might find yourself needing some upstream DataFusion changes not present in the projects current version. Luckily this can easily be achieved by updating [Cargo.toml](Cargo.toml) and changing the `rev` to the SHA of the version you need. Note that the same SHA should be used for all DataFusion modules.
 
 After updating the `Cargo.toml` file the codebase can be re-built to reflect those changes by running `python setup.py install`
 
@@ -72,40 +72,40 @@ Sometimes when building against the latest Github commits for DataFusion you may
 ### Datastructures
 While working in the Rust codebase there are a few datastructures that you should make yourself familiar with. This section does not aim to verbosely list out all of the datastructure with in the project but rather just the key datastructures that you are likely to encounter while working on almost any feature/issue. The aim is to give you a better overview of the codebase without having to manually dig through the all the source code.
 
-- [`PyLogicalPlan`](dask_planner/src/sql/logical.rs) -> [DataFusion LogicalPlan](https://docs.rs/datafusion/latest/datafusion/logical_plan/enum.LogicalPlan.html)
+- [`PyLogicalPlan`](src/sql/logical.rs) -> [DataFusion LogicalPlan](https://docs.rs/datafusion/latest/datafusion/logical_plan/enum.LogicalPlan.html)
     - Often encountered in Python code with variable name `rel`
     - Python serializable umbrella representation of the entire LogicalPlan that was generated by DataFusion
     - Provides access to `DaskTable` instances and type information for each table
     - Access to individual nodes in the logical plan tree. Ex: `TableScan`
-- [`DaskSQLContext`](dask_planner/src/sql.rs)
+- [`DaskSQLContext`](src/sql.rs)
     - Analogous to Python `Context`
     - Contains metadata about the tables, schemas, functions, operators, and configurations that are persent within the current execution context
     - When adding custom functions/UDFs this is the location that you would register them
     - Entry point for parsing SQL strings to sql node trees. This is the location Python will begin its interactions with Rust
-- [`PyExpr`](dask_planner/src/expression.rs) -> [DataFusion Expr](https://docs.rs/datafusion/latest/datafusion/prelude/enum.Expr.html)
+- [`PyExpr`](src/expression.rs) -> [DataFusion Expr](https://docs.rs/datafusion/latest/datafusion/prelude/enum.Expr.html)
     - Arguably where most of your time will be spent
     - Represents a single node in sql tree. Ex: `avg(age)` from `SELECT avg(age) FROM people`
     - Is associate with a single `RexType`
     - Can contain literal values or represent function calls, `avg()` for example
     - The expressions "index" in the tree can be retrieved by calling `PyExpr.index()` on an instance. This is useful when mapping frontend column names in Dask code to backend Dataframe columns
     - Certain `PyExpr`s contain operands. Ex: `2 + 2` would contain 3 operands. 1) A literal `PyExpr` instance with value 2 2) Another literal `PyExpr` instance with a value of 2. 3) A `+` `PyExpr` representing the addition of the 2 literals.
-- [`DaskSqlOptimizer`](dask_planner/src/sql/optimizer.rs)
+- [`DaskSqlOptimizer`](src/sql/optimizer.rs)
     - Registering location for all Dask-SQL specific logical plan optimizations
     - Optimizations that are written either custom or use from another source, DataFusion, are registered here in the order they are wished to be executed
     - Represents functions that modify/convert an original `PyLogicalPlan` into another `PyLogicalPlan` that would be more efficient when running in the underlying Dask framework
-- [`RelDataType`](dask_planner/src/sql/types/rel_data_type.rs)
+- [`RelDataType`](src/sql/types/rel_data_type.rs)
     - Not a fan of this name, was chosen to match existing Calcite logic
     - Represents a "row" in a table
     - Contains a list of "columns" that are present in that row
-        - [RelDataTypeField](dask_planner/src/sql/types/rel_data_type_field.rs)
-- [RelDataTypeField](dask_planner/src/sql/types/rel_data_type_field.rs)
+        - [RelDataTypeField](src/sql/types/rel_data_type_field.rs)
+- [RelDataTypeField](src/sql/types/rel_data_type_field.rs)
     - Represents an individual column in a table
     - Contains:
         - `qualifier` - schema the field belongs to
         - `name` - name of the column/field
         - `data_type` - `DaskTypeMap` instance containing information about the SQL type and underlying Arrow DataType
         - `index` - location of the field in the LogicalPlan
-- [DaskTypeMap](dask_planner/src/sql/types.rs)
+- [DaskTypeMap](src/sql/types.rs)
     - Maps a conventional SQL type to an underlying Arrow DataType