Merge pull request #18 from JasperHG90/feat/lzy

Refactor: add lazy Polars and Daft IO managers without dependency on Arrow
JasperHG90 · Nov 16, 2024 · 1ddd800 · 1ddd800
2 parents c47a01d + b3b303b
commit 1ddd800
Show file tree

Hide file tree

Showing 68 changed files with 1,116 additions and 1,921 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -26,7 +26,7 @@ jobs:
         with:
           python_version: "3.11"
           pre_commit_version: "4.0.1"
-          directory: "."
+          directory: "src"
       - name: 'Pyright type checks'
         uses: ./.github/workflows/templates/pyright_type_checks
         with:

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -26,7 +26,6 @@ jobs:
         with:
           python_version: '3.11'
           uv_version: '0.4.27'
-          path_to_packages: './packages'
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:

diff --git a/.github/workflows/templates/release/build/action.yaml b/.github/workflows/templates/release/build/action.yaml
@@ -10,10 +10,6 @@ inputs:
     description: 'uv version to use'
     required: true
     default: 'latest'
-  path_to_packages:
-    description: 'Path to python packages'
-    required: true
-    default: 'packages'
 
 runs:
   using: 'composite'
@@ -40,10 +36,10 @@ runs:
       uses: ./.github/workflows/templates/release/utils/bump_version
       with:
         version: ${{ steps.version.outputs.version }}
-        root: ${{ inputs.path_to_packages }}
-        glob: '**/src/**/version.py,**/pyproject.toml'
+        root: '.'
+        glob: 'src/**/version.py,**/pyproject.toml'
     - name: 'Build python packages'
       id: build
       run: |
-        uv build --all
+        uv build
       shell: bash
diff --git a/.justfile b/.justfile
@@ -6,7 +6,7 @@ alias d := docs
 
 # Install python dependencies
 install:
-  uv sync
+  uv sync --all-extras
 
 # Install pre-commit hooks
 pre_commit_setup:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -18,7 +18,7 @@ repos:
     rev: "1.7.0"
     hooks:
       - id: interrogate
-        args: [-vv, --fail-under=0, ./packages]
+        args: [-vv, --fail-under=0, ./src]
 
   - repo: https://github.com/ambv/black
     rev: 24.1.1

diff --git a/docs/snippets/io_manager_pandas.py b/docs/snippets/io_manager_pandas.py
@@ -1,8 +1,9 @@
 import pandas as pd
 from dagster import Definitions, asset
-from dagster_pyiceberg import IcebergSqlCatalogConfig
 from dagster_pyiceberg_pandas import IcebergPandasIOManager
 
+from dagster_pyiceberg import IcebergSqlCatalogConfig
+
 CATALOG_URI = "sqlite:////home/vscode/workspace/.tmp/examples/select_columns/catalog.db"
 CATALOG_WAREHOUSE = (
     "file:///home/vscode/workspace/.tmp/examples/select_columns/warehouse"

diff --git a/docs/snippets/io_manager_polars.py b/docs/snippets/io_manager_polars.py
@@ -1,9 +1,10 @@
 import pandas as pd
 import polars as pl
 from dagster import Definitions, asset
-from dagster_pyiceberg import IcebergSqlCatalogConfig
 from dagster_pyiceberg_polars import IcebergPolarsIOManager
 
+from dagster_pyiceberg import IcebergSqlCatalogConfig
+
 CATALOG_URI = "sqlite:////home/vscode/workspace/.tmp/examples/select_columns/catalog.db"
 CATALOG_WAREHOUSE = (
     "file:///home/vscode/workspace/.tmp/examples/select_columns/warehouse"

diff --git a/docs/snippets/io_manager_pyarrow.py b/docs/snippets/io_manager_pyarrow.py
@@ -1,6 +1,7 @@
 import pandas as pd
 import pyarrow as pa
 from dagster import Definitions, asset
+
 from dagster_pyiceberg import IcebergPyarrowIOManager, IcebergSqlCatalogConfig
 
 CATALOG_URI = "sqlite:////home/vscode/workspace/.tmp/examples/select_columns/catalog.db"

diff --git a/docs/snippets/multiple_io_managers.py b/docs/snippets/multiple_io_managers.py
@@ -1,8 +1,9 @@
 import pandas as pd
 from dagster import Definitions, FilesystemIOManager, asset
-from dagster_pyiceberg import IcebergSqlCatalogConfig
 from dagster_pyiceberg_pandas import IcebergPandasIOManager
 
+from dagster_pyiceberg import IcebergSqlCatalogConfig
+
 CATALOG_URI = "sqlite:////home/vscode/workspace/.tmp/examples/catalog.db"
 CATALOG_WAREHOUSE = "file:///home/vscode/workspace/.tmp/examples/warehouse"
 FS_BASE_DIR = "/home/vscode/workspace/.tmp/examples/images"

diff --git a/docs/snippets/multiple_schemas.py b/docs/snippets/multiple_schemas.py
@@ -1,8 +1,9 @@
 import pandas as pd
 from dagster import Definitions, asset
-from dagster_pyiceberg import IcebergSqlCatalogConfig
 from dagster_pyiceberg_pandas import IcebergPandasIOManager
 
+from dagster_pyiceberg import IcebergSqlCatalogConfig
+
 CATALOG_URI = "sqlite:////home/vscode/workspace/.tmp/examples/catalog.db"
 CATALOG_WAREHOUSE = "file:///home/vscode/workspace/.tmp/examples/warehouse"
 

diff --git a/docs/snippets/partitions_multiple.py b/docs/snippets/partitions_multiple.py
@@ -9,9 +9,10 @@
     StaticPartitionDefinition,
     asset,
 )
-from dagster_pyiceberg import IcebergSqlCatalogConfig
 from dagster_pyiceberg_pandas import IcebergPandasIOManager
 
+from dagster_pyiceberg import IcebergSqlCatalogConfig
+
 CATALOG_URI = "sqlite:////home/vscode/workspace/.tmp/examples/catalog.db"
 CATALOG_WAREHOUSE = "file:///home/vscode/workspace/.tmp/examples/warehouse"
 

diff --git a/docs/snippets/partitions_static.py b/docs/snippets/partitions_static.py
@@ -1,8 +1,9 @@
 import pandas as pd
 from dagster import Definitions, StaticPartitionsDefinition, asset
-from dagster_pyiceberg import IcebergSqlCatalogConfig
 from dagster_pyiceberg_pandas import IcebergPandasIOManager
 
+from dagster_pyiceberg import IcebergSqlCatalogConfig
+
 CATALOG_URI = "sqlite:////home/vscode/workspace/.tmp/examples/catalog.db"
 CATALOG_WAREHOUSE = "file:///home/vscode/workspace/.tmp/examples/warehouse"
 

diff --git a/docs/snippets/partitions_time.py b/docs/snippets/partitions_time.py
@@ -3,9 +3,10 @@
 
 import pandas as pd
 from dagster import DailyPartitionsDefinition, Definitions, asset
-from dagster_pyiceberg import IcebergSqlCatalogConfig
 from dagster_pyiceberg_pandas import IcebergPandasIOManager
 
+from dagster_pyiceberg import IcebergSqlCatalogConfig
+
 CATALOG_URI = "sqlite:////home/vscode/workspace/.tmp/examples/catalog.db"
 CATALOG_WAREHOUSE = "file:///home/vscode/workspace/.tmp/examples/warehouse"
 

diff --git a/docs/snippets/pyiceberg_resource.py b/docs/snippets/pyiceberg_resource.py
@@ -4,6 +4,7 @@
 
 import pandas as pd
 from dagster import Definitions, asset
+
 from dagster_pyiceberg import IcebergSqlCatalogConfig, IcebergTableResource
 
 CATALOG_URI = "sqlite:////home/vscode/workspace/.tmp/examples/catalog.db"

diff --git a/docs/snippets/select_columns.py b/docs/snippets/select_columns.py
@@ -1,8 +1,9 @@
 import pandas as pd
 from dagster import AssetIn, Definitions, asset
-from dagster_pyiceberg import IcebergSqlCatalogConfig
 from dagster_pyiceberg_pandas import IcebergPandasIOManager
 
+from dagster_pyiceberg import IcebergSqlCatalogConfig
+
 CATALOG_URI = "sqlite:////home/vscode/workspace/.tmp/examples/select_columns/catalog.db"
 CATALOG_WAREHOUSE = (
     "file:///home/vscode/workspace/.tmp/examples/select_columns/warehouse"

diff --git a/docs/snippets/table_properties.py b/docs/snippets/table_properties.py
@@ -1,8 +1,9 @@
 import pandas as pd
 from dagster import Definitions, asset
-from dagster_pyiceberg import IcebergSqlCatalogConfig
 from dagster_pyiceberg_pandas import IcebergPandasIOManager
 
+from dagster_pyiceberg import IcebergSqlCatalogConfig
+
 CATALOG_URI = "sqlite:////home/vscode/workspace/.tmp/examples/select_columns/catalog.db"
 CATALOG_WAREHOUSE = (
     "file:///home/vscode/workspace/.tmp/examples/select_columns/warehouse"

diff --git a/packages/dagster_pyiceberg/LICENSE.txt b/packages/dagster_pyiceberg/LICENSE.txt
diff --git a/packages/dagster_pyiceberg/README.md b/packages/dagster_pyiceberg/README.md