From 8c96ab1f89afe6233e37f93297ac095a742bddbd Mon Sep 17 00:00:00 2001 From: Github Action Date: Mon, 11 Mar 2024 21:22:22 +0000 Subject: [PATCH] doc update for tag `main` --- 404.html | 22 + api/catalog/index.html | 22 + .../delta_table_alterer/index.html | 22 + api/delta_table/delta_table_merger/index.html | 63 +- .../delta_table_optimizer/index.html | 22 + api/delta_table/index.html | 22 + api/delta_table/metadata/index.html | 22 + api/delta_writer/index.html | 22 + api/exceptions/index.html | 22 + api/schema/index.html | 22 + api/storage/index.html | 22 + delta-lake-best-practices/index.html | 22 + delta-lake-big-data-small-data/index.html | 22 + .../architecture-of-delta-table/index.html | 22 + .../delta-lake-acid-transactions/index.html | 22 + .../delta-lake-file-skipping/index.html | 22 + index.html | 22 + integrations/delta-lake-arrow/index.html | 28 +- integrations/delta-lake-dagster/index.html | 2159 +++++++++++++++++ integrations/delta-lake-dask/index.html | 28 +- integrations/delta-lake-datafusion/index.html | 22 + integrations/delta-lake-pandas/index.html | 22 + integrations/delta-lake-polars/index.html | 22 + search/search_index.json | 2 +- sitemap.xml | 77 +- sitemap.xml.gz | Bin 586 -> 592 bytes .../index.html | 22 + usage/constraints/index.html | 22 + usage/create-delta-lake-table/index.html | 22 + .../index.html | 22 + usage/examining-table/index.html | 22 + usage/installation/index.html | 22 + usage/loading-table/index.html | 22 + usage/managing-tables/index.html | 22 + usage/optimize/delta-lake-z-order/index.html | 22 + .../index.html | 22 + usage/overview/index.html | 22 + usage/querying-delta-tables/index.html | 22 + usage/writing/index.html | 22 + .../index.html | 22 + why-use-delta-lake/index.html | 22 + 41 files changed, 3059 insertions(+), 46 deletions(-) create mode 100644 integrations/delta-lake-dagster/index.html diff --git a/404.html b/404.html index ae2fee5b81..6166bd7fff 100644 --- a/404.html +++ b/404.html @@ -1265,6 +1265,8 @@ + + @@ -1328,6 +1330,26 @@ +
  • + + + + + Dagster + + + + +
    • + + + + + + + + +
  • diff --git a/api/catalog/index.html b/api/catalog/index.html index e37bf28d70..08adb8cc7d 100644 --- a/api/catalog/index.html +++ b/api/catalog/index.html @@ -1341,6 +1341,8 @@ + + @@ -1404,6 +1406,26 @@ +
  • + + + + + Dagster + + + + +
    • + + + + + + + + +
  • diff --git a/api/delta_table/delta_table_alterer/index.html b/api/delta_table/delta_table_alterer/index.html index fadb50c6c1..a0e02f165d 100644 --- a/api/delta_table/delta_table_alterer/index.html +++ b/api/delta_table/delta_table_alterer/index.html @@ -1345,6 +1345,8 @@ + + @@ -1408,6 +1410,26 @@ +
  • + + + + + Dagster + + + + +
    • + + + + + + + + +
  • diff --git a/api/delta_table/delta_table_merger/index.html b/api/delta_table/delta_table_merger/index.html index 5c3bf40d75..e2c6d53920 100644 --- a/api/delta_table/delta_table_merger/index.html +++ b/api/delta_table/delta_table_merger/index.html @@ -1394,6 +1394,8 @@ + + @@ -1457,6 +1459,26 @@ +
  • + + + + + Dagster + + + + +
    • + + + + + + + + +
  • @@ -1871,6 +1893,11 @@

    Delete a matched row from the table only if the given predicate (if specified) is true for the matched row. If not specified it deletes all matches.

    +
    + Note +

    Column names with special characters, such as numbers or spaces should be encapsulated +in backticks: "target.123column" or "target.my column"

    +

    Parameters:

    @@ -1993,6 +2020,11 @@

    Update a matched table row based on the rules defined by updates. If a predicate is specified, then it must evaluate to true for the row to be updated.

    +
    + Note +

    Column names with special characters, such as numbers or spaces should be encapsulated +in backticks: "target.123column" or "target.my column"

    +

    Parameters:

    @@ -2066,10 +2098,10 @@ 

    from deltalake import DeltaTable, write_deltalake
 import pyarrow as pa
 
-data = pa.table({"x": [1, 2, 3], "y": [4, 5, 6]})
+data = pa.table({"x": [1, 2, 3], "1y": [4, 5, 6]})
 write_deltalake("tmp", data)
 dt = DeltaTable("tmp")
-new_data = pa.table({"x": [1], "y": [7]})
+new_data = pa.table({"x": [1], "1y": [7]})
 
 (
      dt.merge(
@@ -2077,7 +2109,7 @@ 
             predicate="target.x = source.x",
          source_alias="source",
          target_alias="target")
-     .when_matched_update(updates={"x": "source.x", "y": "source.y"})
+     .when_matched_update(updates={"x": "source.x", "`1y`": "source.`1y`"})
      .execute()
 )
 {'num_source_rows': 1, 'num_target_rows_inserted': 0, 'num_target_rows_updated': 1, 'num_target_rows_deleted': 0, 'num_target_rows_copied': 2, 'num_output_rows': 3, 'num_target_files_added': 1, 'num_target_files_removed': 1, 'execution_time_ms': ..., 'scan_time_ms': ..., 'rewrite_time_ms': ...}
@@ -2111,6 +2143,11 @@ 
    
+  Note
+  
    Column names with special characters, such as numbers or spaces should be encapsulated
+in backticks: "target.123column" or "target.my column"
    
+
 
 
   
    Parameters:
    
@@ -2215,6 +2252,11 @@ 
    
+  Note
+  
    Column names with special characters, such as numbers or spaces should be encapsulated
+in backticks: "target.123column" or "target.my column"
    
+
 
 
   
    Parameters:
    
@@ -2291,6 +2333,11 @@ 
    
+  Note
+  
    Column names with special characters, such as numbers or spaces should be encapsulated
+in backticks: "target.123column" or "target.my column"
    
+
 
 
   
    Parameters:
    
@@ -2411,6 +2458,11 @@ 
    
+  Note
+  
    Column names with special characters, such as numbers or spaces should be encapsulated
+in backticks: "target.123column" or "target.my column"
    
+
 
 
   
    Parameters:
    
@@ -2535,6 +2587,11 @@ 
    
+  Note
+  
    Column names with special characters, such as numbers or spaces should be encapsulated
+in backticks: "target.123column" or "target.my column"
    
+
 
 
   
    Parameters:
    
diff --git a/api/delta_table/delta_table_optimizer/index.html b/api/delta_table/delta_table_optimizer/index.html
index e9b3ffce53..75a90a7d4e 100644
--- a/api/delta_table/delta_table_optimizer/index.html
+++ b/api/delta_table/delta_table_optimizer/index.html
@@ -1345,6 +1345,8 @@
         
       
         
+      
+        
       
         
       
@@ -1408,6 +1410,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Dagster
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/api/delta_table/index.html b/api/delta_table/index.html
index 11d6821136..372f4d2a46 100644
--- a/api/delta_table/index.html
+++ b/api/delta_table/index.html
@@ -1278,6 +1278,8 @@
         
       
         
+      
+        
       
         
       
@@ -1341,6 +1343,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Dagster
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/api/delta_table/metadata/index.html b/api/delta_table/metadata/index.html
index b330f9a411..a2fbbe2138 100644
--- a/api/delta_table/metadata/index.html
+++ b/api/delta_table/metadata/index.html
@@ -1373,6 +1373,8 @@
         
       
         
+      
+        
       
         
       
@@ -1436,6 +1438,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Dagster
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/api/delta_writer/index.html b/api/delta_writer/index.html
index 2253dc2638..054295fe52 100644
--- a/api/delta_writer/index.html
+++ b/api/delta_writer/index.html
@@ -1365,6 +1365,8 @@
         
       
         
+      
+        
       
         
       
@@ -1428,6 +1430,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Dagster
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/api/exceptions/index.html b/api/exceptions/index.html
index 28495c05f8..fbf7573431 100644
--- a/api/exceptions/index.html
+++ b/api/exceptions/index.html
@@ -1346,6 +1346,8 @@
         
       
         
+      
+        
       
         
       
@@ -1409,6 +1411,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Dagster
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/api/schema/index.html b/api/schema/index.html
index 5c48c1ea09..977c374558 100644
--- a/api/schema/index.html
+++ b/api/schema/index.html
@@ -1684,6 +1684,8 @@
         
       
         
+      
+        
       
         
       
@@ -1747,6 +1749,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Dagster
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/api/storage/index.html b/api/storage/index.html
index b851599a4c..8ac776b060 100644
--- a/api/storage/index.html
+++ b/api/storage/index.html
@@ -1359,6 +1359,8 @@
         
       
         
+      
+        
       
         
       
@@ -1422,6 +1424,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Dagster
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/delta-lake-best-practices/index.html b/delta-lake-best-practices/index.html
index 4fd99924b9..2bb629ba94 100644
--- a/delta-lake-best-practices/index.html
+++ b/delta-lake-best-practices/index.html
@@ -1395,6 +1395,8 @@
         
       
         
+      
+        
       
         
       
@@ -1458,6 +1460,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Dagster
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/delta-lake-big-data-small-data/index.html b/delta-lake-big-data-small-data/index.html
index 87d673b725..29f83e235c 100644
--- a/delta-lake-big-data-small-data/index.html
+++ b/delta-lake-big-data-small-data/index.html
@@ -1353,6 +1353,8 @@
         
       
         
+      
+        
       
         
       
@@ -1416,6 +1418,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Dagster
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/how-delta-lake-works/architecture-of-delta-table/index.html b/how-delta-lake-works/architecture-of-delta-table/index.html
index cfadda57c4..cad00fd4b2 100644
--- a/how-delta-lake-works/architecture-of-delta-table/index.html
+++ b/how-delta-lake-works/architecture-of-delta-table/index.html
@@ -1278,6 +1278,8 @@
         
       
         
+      
+        
       
         
       
@@ -1341,6 +1343,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Dagster
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/how-delta-lake-works/delta-lake-acid-transactions/index.html b/how-delta-lake-works/delta-lake-acid-transactions/index.html
index 51331c3e47..41b9eca812 100644
--- a/how-delta-lake-works/delta-lake-acid-transactions/index.html
+++ b/how-delta-lake-works/delta-lake-acid-transactions/index.html
@@ -1278,6 +1278,8 @@
         
       
         
+      
+        
       
         
       
@@ -1341,6 +1343,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Dagster
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/how-delta-lake-works/delta-lake-file-skipping/index.html b/how-delta-lake-works/delta-lake-file-skipping/index.html
index b37c711b52..24493431dd 100644
--- a/how-delta-lake-works/delta-lake-file-skipping/index.html
+++ b/how-delta-lake-works/delta-lake-file-skipping/index.html
@@ -1276,6 +1276,8 @@
         
       
         
+      
+        
       
         
       
@@ -1339,6 +1341,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Dagster
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/index.html b/index.html
index 31d47818dc..99c6f808a3 100644
--- a/index.html
+++ b/index.html
@@ -1276,6 +1276,8 @@
         
       
         
+      
+        
       
         
       
@@ -1339,6 +1341,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Dagster
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/integrations/delta-lake-arrow/index.html b/integrations/delta-lake-arrow/index.html
index d4fdcb1309..08271eef76 100644
--- a/integrations/delta-lake-arrow/index.html
+++ b/integrations/delta-lake-arrow/index.html
@@ -14,7 +14,7 @@
         
       
       
-        
+        
       
       
       
@@ -1280,6 +1280,8 @@
         
       
         
+      
+        
       
         
       
@@ -1416,6 +1418,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Dagster
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
@@ -1806,13 +1828,13 @@ 
    Conclusion
    
         
         
           
-              
+          
             
    
               
                 Next
               
               
    
-                Dask
+                Dagster
               
    
             
    
             
    
diff --git a/integrations/delta-lake-dagster/index.html b/integrations/delta-lake-dagster/index.html
new file mode 100644
index 0000000000..79c0286211
--- /dev/null
+++ b/integrations/delta-lake-dagster/index.html
@@ -0,0 +1,2159 @@
+
+
+
+  
+    
+      
+      
+      
+      
+      
+        
+      
+      
+        
+      
+      
+        
+      
+      
+      
+      
+    
+    
+      
+        Dagster - Delta Lake Documentation
+      
+    
+    
+      
+      
+        
+        
+      
+      
+
+
+    
+    
+      
+    
+    
+      
+        
+        
+        
+        
+        
+      
+    
+    
+      
+    
+      
+    
+    
+    
+      
+
+    
+    
+    
+  
+  
+  
+    
+    
+      
+    
+    
+    
+    
+    
+  
+    
+    
+      
+    
+    
+    
+    
+    
+    
    
+      
+    
    
+    
+    
+      
+
+  
+
+
    
+  
+  
+    
+      
+
+    
+  
+
    
+    
+    
    
+      
+      
+        
+      
+      
    
+        
    
+          
+            
+              
+              
    
+                
    
+                  
    
+                    
+
+
+  
+
+
+
+                  
    
+                
    
+              
    
+            
+            
+              
+              
    
+                
    
+                  
    
+                    
+
+
+                  
    
+                
    
+              
    
+            
+          
+          
+            
    
+              
    
+                
+                  
+
+  
+  
+
+
+
    Using Delta Lake with Dagster¶
    
+
    Delta Lake is a great storage format for Dagster workflows. This page will explain why and how to use Delta Lake with Dagster.
    
+
    You will learn how to use the Delta Lake I/O Manager to read and write your Dagster Software-Defined Assets (SDAs). You will also learn about the unique advantages Delta Lake offers the Dagster community.
    
+
    Here are some of the benefits that Delta Lake provides Dagster users:
+- native PyArrow integration for lazy computation of large datasets,
+- more efficient querying with file skipping via Z Ordering and liquid clustering
+- built-in vacuuming to remove unnecessary files and versions
+- ACID transactions for reliable writes
+- smooth versioning integration so that versions can be use to trigger downstream updates.
+- surfacing table stats based on the file statistics 
    
+
    Dagster I/O Managers
    
+
    Dagster uses I/O Managers to simplify data reads and writes. I/O Managers help you reduce boilerplate code by storing Dagster Asset and Op outputs and loading them as inputs to downstream objects. They make it easy to change where and how your data is stored.
    
+
    You only need to define your I/O Manager and its settings (such as storage location and schema) once and the I/O Manager will take care of correctly reading and writing all your Dagster Assets automatically.
    
+
    If you need lower-level access than the Dagster I/O Managers provide, take a look at the Delta Table Resource.
    
+
    The Delta Lake I/O Manager
    
+
    You can easily read and write Delta Lake Tables from Dagster by using the DeltaLakeIOManager(). 
    
+
    Install the DeltaLakeIOManager:
    
+
    pip install dagster-deltalake
+
    
+
    Next, configure the following settings in your project’s __init__.py file:
+- io_manager: set this to DeltaLakeIOManager(), this sets the default I/O Manager for all your Assets
    
+
    Within the DeltaLakeIOManager, define:
+- root_uri: the root path where your Delta Tables will be created
+- storage_options: configuration for accessing storage location
+- schema: name of schema to use (optional, defaults to public)
    
+
    defs = Definitions(
+   assets=all_assets,
+   resources={
+        "io_manager": DeltaLakePyarrowIOManager(
+            root_uri="path/to/deltalake",
+            storage_options=LocalConfig(),
+            schema="dagster_deltalake",
+        ),
+   },
+)
+
    
+
    Now, when you materialize an Asset, it will be saved as a Delta Lake in a folder dagster_deltalake/asset_name under the root directory path/to/deltalake.
    
+
    The default Delta Lake I/O Manager supports Arrow reads and writes. You can also use the Delta Lake I/O Manager with pandas or polars. 
    
+
    Creating Delta Lake Tables with Dagster
    
+
    You don’t need to do anything else to store your Dagster Assets as Delta Lake tables. The I/O Manager will handle storing and loading your Assets as Delta Lake tables from now on.
    
+
    You can proceed to write Dagster code as you normally would. For example, you can create an Asset that reads in some toy data about animals and writes it out to an Arrow Table:
    
+
    import pyarrow as pa
+from pyarrow import csv
+
+from dagster import asset
+
+@asset
+def raw_dataset() -> pa.Table:
+   n_legs = pa.array([2, 4, None, 100])
+   animals = pa.array(["Flamingo", "Horse", "Brittle stars", "Centipede"])
+   data = {'n_legs': n_legs, 'animals': animals}
+
+   return pa.Table.from_pydict(data)
+
    
+
    When you materialize the Asset defined above (using the config settings defined earlier), the Delta Lake I/O Manager will create the table dagster_deltalake/iris_dataset if it doesn’t exist yet.
    
+
    Overwrites when Rematerializing Assets
    
+
    If the table does already exist at the specified location, the Delta Lake I/O Manager will perform an overwrite. Delta Lake’s transaction log maintains a record of all changes to your Delta Lake tables. You can inspect the record of changes to your Delta Lake tables by taking a look at these transaction logs.
    
+
    Loading Delta Lake Tables in Downstream Assets
    
+
    You can use Assets stored as Delta Lake tables as input to downstream Assets. Dagster and the Delta Lake I/O Manager make this easy for you.
    
+
    You can write Dagster code as you normally would. Pass the upstream Asset as an argument to the downstream object to set up the dependency. Make sure to define the correct data type.
    
+
    The Delta Lake I/O Manager will handle reading and writing the data from your Delta Lake.
    
+
    import pyarrow as pa
+from dagster import asset
+
+# ... raw_dataset asset is defined here ...
+
+@asset
+def clean_dataset(raw_dataset: pa.Table) -> pa.Table:
+   return raw_dataset.drop_null()
+
    
+
    Reading Existing Delta Lake Tables into Dagster
    
+
    You can make existing Delta Lake tables (that were not created in Dagster) available to your Dagster assets. Use the SourceAsset object and pass the table name as the key argument: 
    
+
    from dagster import SourceAsset
+
+iris_harvest_data = SourceAsset(key="more_animal_data")
+
    
+
    This will load a table more_animal_data located at <root_uri>/<schema> as configured in the Definitions object above (see Delta Lake I/O Manager section).
    
+
    Column Pruning
    
+
    You can often improve the efficiency of your computations by only loading specific columns of your Delta table. This is called column pruning. 
    
+
    With the Delta Lake I/O manager, you can select specific columns to load defining the columns in the metadata parameter of the AssetIn that loads the upstream Asset:
    
+
    import pyarrow as pa
+from dagster import AssetIn, asset
+
+# this example uses the clean_dataset Asset defined earlier
+
+@asset(
+       ins={
+           "mammal_bool": AssetIn(
+               key="clean_dataset",
+               metadata={"columns": ["is_mammal", "animals"]},
+           )
+       }
+)
+def mammal_data(mammal_bool: pa.Table) -> pa.Table:
+   mammals = mammal_bool["is_mammal"].cast("bool")
+   animals = mammal_bool["animals"]
+   data = {"mammal_bool": mammals, "animals": animals}
+   return pa.Table.from_pydict(data)
+
    
+
    Here, we select only the sepal_length_cm and sepal_width_cm columns from the iris_dataset table and load them into an AssetIn object called iris_sepal. This AssetIn object is used to create a new Asset sepal_data, containing only the selected columns.
    
+
    Working with Partitioned Assets
    
+
    Partitioning is an important feature of Delta Lake that can make your computations more efficient. The Delta Lake I/O manager helps you read and write partitioned data easily. You can work with static partitions, time-based partitions, multi-partitions, and dynamic partitions.
    
+
    For example, you can partition the Iris dataset on the species column as follows:
    
+
    import pyarrow as pa
+
+from dagster import StaticPartitionsDefinition, asset
+
+@asset(
+  partitions_def=StaticPartitionsDefinition(
+      ["Human", "Horse",]
+  ),
+  metadata={"partition_expr": "n_legs"},
+)
+def dataset_partitioned(
+   context,
+   clean_dataset: pa.Table,
+   ) -> pa.Table:
+   animals = context.asset_partition_key_for_output()
+   table = clean_dataset
+
+   return table.filter(pc.field("animals") == animals)
+
    
+
    To partition your data, make sure to include the relevant partitions_def and metadata arguments to the @asset decorator. Refer to the Dagster documentation on partitioning assets for more information.
    
+
    Using Delta Lake and Dagster with Pandas
    
+
    To read and write data to Delta Lake using pandas, use the DeltaLakePandasIOManager(). 
    
+
    You will need to install it using:
    
+
    pip install dagster-deltalake-pandas
+
    
+
    In your Definitions object, change the io_manager to DeltaLakePandasIOManager():
    
+
    from dagster_deltalake_pandas import DeltaLakePandasIOManager
+
+
+defs = Definitions(
+   assets=all_assets,
+   resources={
+        "io_manager": DeltaLakePandasIOManager(
+            root_uri="path/to/deltalake",
+            storage_options=LocalConfig(),
+            schema="dagster_deltalake",
+        ),
+   },
+)
+
    
+
    Now you can read and write Dagster Assets defined as pandas DataFrames in Delta Lake format. For example:
    
+
    import pandas as pd
+from dagster import asset
+
+@asset
+def iris_dataset() -> pd.DataFrame:
+   return pd.read_csv(
+       "https://docs.dagster.io/assets/iris.csv",
+       names=[
+           "sepal_length_cm",
+           "sepal_width_cm",
+           "petal_length_cm",
+           "petal_width_cm",
+           "species",
+       ],
+   )
+
    
+
    Using Delta Lake and Dagster with Polars
    
+
    To read and write data to Delta Lake using pandas, use the DeltaLakePolarsIOManager(). 
    
+
    You will need to install it using:
    
+
    pip install dagster-deltalake-polars
+
    
+
    In your Definitions object, change the io_manager to DeltaLakePolarsIOManager():
    
+
    from dagster_polars import DeltaLakePolarsIOManager
+
+defs = Definitions(
+   assets=all_assets,
+   resources={
+        "io_manager": DeltaLakePolarsIOManager(
+            root_uri="path/to/deltalake",
+            storage_options=LocalConfig(),
+            schema="dagster_deltalake",
+        ),
+   },
+)
+
    
+
    Now you can read and write Dagster Assets defined as Polars DataFrames in Delta Lake format. For example:
    
+
    import polars as pl
+from dagster import asset
+
+
+@asset
+def iris_dataset() -> pl.DataFrame:
+   return pl.read_csv(
+       "https://docs.dagster.io/assets/iris.csv",
+       new_columns=[
+          "sepal_length_cm",
+          "sepal_width_cm",
+          "petal_length_cm",
+          "petal_width_cm",
+          "species",
+      ],
+   has_header=False
+)
+
    
+
    Delta Lake Table Resource
    
+
    I/O managers are a helpful tool in many common usage situations. But when you need lower-level access, the I/O Manager might not be the right tool to use. In these cases you may want to use the Delta Lake Table Resource.
    
+
    The Delta Lake Table Resource is a low-level access method to the table object. It gives you more fine-grained control and allows for modeling of more complex data. You can also use the Table Resource to run optimization and vacuuming jobs.
    
+
    Schema and Constraint Enforcement
    
+
    Delta Lake provides built-in checks to ensure schema consistency when appending data to a table, as well as the ability to evolve the schema. This is a great feature for the Dagster community as it prevents bad data from being appended to tables, ensuring data consistency and accuracy. 
    
+
    Read more about how to add constraints to a table in the Delta Lake documentation.
    
+
    Z-Ordering
    
+
    Delta Lake offers Z-ordering functionality to colocate similar data in the same files. This can make your Delta Table queries much more efficient via file skipping. Dagster users can now benefit from this great feature through the Delta Lake I/O Manager.
    
+
    Read more about Z-Ordering on the Delta Lake blog.
    
+
    Contribute
    
+
    To contribute to the Delta Lake and Dagster integration, go to [link]
    
+
+
+
+
+
+                
+              
    
+            
    
+          
+          
+  
+
+        
    
+        
+      
    
+      
+        
+      
+    
    
+    
    
+      
    
+    
    
+    
+    
+    
+    
+    
+      
+      
+    
+  
+
\ No newline at end of file
diff --git a/integrations/delta-lake-dask/index.html b/integrations/delta-lake-dask/index.html
index ec1a2f0fa8..275735c800 100644
--- a/integrations/delta-lake-dask/index.html
+++ b/integrations/delta-lake-dask/index.html
@@ -11,7 +11,7 @@
         
       
       
-        
+        
       
       
         
@@ -1280,6 +1280,8 @@
         
       
         
+      
+        
       
         
       
@@ -1340,6 +1342,26 @@
                 
   
   
+  
+    
  • 
+      
+        
+  
+  
+    Dagster
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
     
   
   
@@ -1871,7 +1893,7 @@ 
    Contribute to dask-deltalake<