From 47d06235a9d04bfad3789ab4a27ee39b4add210f Mon Sep 17 00:00:00 2001 From: Github Action Date: Sat, 13 Apr 2024 14:16:58 +0000 Subject: [PATCH] doc update for tag `main` --- 404.html | 22 + api/catalog/index.html | 22 + .../delta_table_alterer/index.html | 28 +- api/delta_table/delta_table_merger/index.html | 42 +- .../delta_table_optimizer/index.html | 28 +- api/delta_table/index.html | 72 +- api/delta_table/metadata/index.html | 24 +- api/delta_writer/index.html | 28 +- api/exceptions/index.html | 22 + api/schema/index.html | 80 +- api/storage/index.html | 32 +- assets/_mkdocstrings.css | 7 +- delta-lake-best-practices/index.html | 22 + delta-lake-big-data-small-data/index.html | 22 + .../architecture-of-delta-table/index.html | 22 + .../delta-lake-acid-transactions/index.html | 22 + .../delta-lake-file-skipping/index.html | 22 + index.html | 22 + integrations/delta-lake-arrow/index.html | 28 +- integrations/delta-lake-daft/index.html | 2156 +++++++++++++++++ integrations/delta-lake-dagster/index.html | 28 +- integrations/delta-lake-dask/index.html | 22 + integrations/delta-lake-datafusion/index.html | 22 + integrations/delta-lake-pandas/index.html | 22 + integrations/delta-lake-polars/index.html | 22 + search/search_index.json | 2 +- sitemap.xml | 79 +- sitemap.xml.gz | Bin 592 -> 597 bytes .../index.html | 22 + usage/constraints/index.html | 22 + usage/create-delta-lake-table/index.html | 22 + .../index.html | 22 + usage/examining-table/index.html | 22 + usage/installation/index.html | 22 + usage/loading-table/index.html | 22 + usage/managing-tables/index.html | 22 + usage/optimize/delta-lake-z-order/index.html | 22 + .../index.html | 22 + usage/overview/index.html | 22 + usage/querying-delta-tables/index.html | 22 + usage/writing/index.html | 22 + .../index.html | 47 + why-use-delta-lake/index.html | 22 + 43 files changed, 3151 insertions(+), 124 deletions(-) create mode 100644 integrations/delta-lake-daft/index.html diff --git a/404.html b/404.html index 6166bd7fff..f9c2683f01 100644 --- a/404.html +++ b/404.html @@ -1267,6 +1267,8 @@ + + @@ -1330,6 +1332,26 @@ +
  • + + + + + Daft + + + + +
    • + + + + + + + + +
  • diff --git a/api/catalog/index.html b/api/catalog/index.html index 08adb8cc7d..752f112b6b 100644 --- a/api/catalog/index.html +++ b/api/catalog/index.html @@ -1343,6 +1343,8 @@ + + @@ -1406,6 +1408,26 @@ +
  • + + + + + Daft + + + + +
    • + + + + + + + + +
  • diff --git a/api/delta_table/delta_table_alterer/index.html b/api/delta_table/delta_table_alterer/index.html index a0e02f165d..f1d95f7863 100644 --- a/api/delta_table/delta_table_alterer/index.html +++ b/api/delta_table/delta_table_alterer/index.html @@ -1347,6 +1347,8 @@ + + @@ -1410,6 +1412,26 @@ +
  • + + + + + Daft + + + + +
    • + + + + + + + + +
  • @@ -1711,7 +1733,7 @@

    -
    TableAlterer(table: DeltaTable)
+
    TableAlterer(table: DeltaTable)
 
    
 
   
    
@@ -1742,7 +1764,7 @@ 
    
 
 
 
    
-
    add_constraint(constraints: Dict[str, str], custom_metadata: Optional[Dict[str, str]] = None) -> None
+
    add_constraint(constraints: Dict[str, str], custom_metadata: Optional[Dict[str, str]] = None) -> None
 
    
 
   
    
@@ -1820,7 +1842,7 @@ 
    
 
 
 
    
-
    drop_constraint(name: str, raise_if_not_exists: bool = True, custom_metadata: Optional[Dict[str, str]] = None) -> None
+
    drop_constraint(name: str, raise_if_not_exists: bool = True, custom_metadata: Optional[Dict[str, str]] = None) -> None
 
    
 
   
    
diff --git a/api/delta_table/delta_table_merger/index.html b/api/delta_table/delta_table_merger/index.html
index e2c6d53920..a6f8dfb781 100644
--- a/api/delta_table/delta_table_merger/index.html
+++ b/api/delta_table/delta_table_merger/index.html
@@ -1396,6 +1396,8 @@
         
       
         
+      
+        
       
         
       
@@ -1459,6 +1461,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Daft
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
@@ -1809,7 +1831,7 @@ 
    
 
 
 
    
-
    TableMerger(table: DeltaTable, source: pyarrow.RecordBatchReader, predicate: str, source_alias: Optional[str] = None, target_alias: Optional[str] = None, safe_cast: bool = True, writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None)
+
    TableMerger(table: DeltaTable, source: pyarrow.RecordBatchReader, predicate: str, source_alias: Optional[str] = None, target_alias: Optional[str] = None, safe_cast: bool = True, writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None)
 
    
 
   
    
@@ -1840,7 +1862,7 @@ 
    
 
 
 
    
-
    execute() -> Dict[str, Any]
+
    execute() -> Dict[str, Any]
 
    
 
   
    
@@ -1885,7 +1907,7 @@ 
    
-
    when_matched_delete(predicate: Optional[str] = None) -> TableMerger
+
    when_matched_delete(predicate: Optional[str] = None) -> TableMerger
 
    
 
   
    
@@ -2012,7 +2034,7 @@ 
    
-
    when_matched_update(updates: Dict[str, str], predicate: Optional[str] = None) -> TableMerger
+
    when_matched_update(updates: Dict[str, str], predicate: Optional[str] = None) -> TableMerger
 
    
 
   
    
@@ -2135,7 +2157,7 @@ 
    when_matched_update_all(predicate: Optional[str] = None) -> TableMerger
+
    when_matched_update_all(predicate: Optional[str] = None) -> TableMerger
 
    
 
   
    
@@ -2244,7 +2266,7 @@ 
    when_not_matched_by_source_delete(predicate: Optional[str] = None) -> TableMerger
+
    when_not_matched_by_source_delete(predicate: Optional[str] = None) -> TableMerger
 
    
 
   
    
@@ -2325,7 +2347,7 @@ 
    when_not_matched_by_source_update(updates: Dict[str, str], predicate: Optional[str] = None) -> TableMerger
+
    when_not_matched_by_source_update(updates: Dict[str, str], predicate: Optional[str] = None) -> TableMerger
 
    
 
   
    
@@ -2450,7 +2472,7 @@ 
    when_not_matched_insert(updates: Dict[str, str], predicate: Optional[str] = None) -> TableMerger
+
    when_not_matched_insert(updates: Dict[str, str], predicate: Optional[str] = None) -> TableMerger
 
    
 
   
    
@@ -2578,7 +2600,7 @@ 
    when_not_matched_insert_all(predicate: Optional[str] = None) -> TableMerger
+
    when_not_matched_insert_all(predicate: Optional[str] = None) -> TableMerger
 
    
 
   
    
@@ -2689,7 +2711,7 @@ 
    with_writer_properties(data_page_size_limit: Optional[int] = None, dictionary_page_size_limit: Optional[int] = None, data_page_row_count_limit: Optional[int] = None, write_batch_size: Optional[int] = None, max_row_group_size: Optional[int] = None) -> TableMerger
+
    with_writer_properties(data_page_size_limit: Optional[int] = None, dictionary_page_size_limit: Optional[int] = None, data_page_row_count_limit: Optional[int] = None, write_batch_size: Optional[int] = None, max_row_group_size: Optional[int] = None) -> TableMerger
 
    
 
   
    
diff --git a/api/delta_table/delta_table_optimizer/index.html b/api/delta_table/delta_table_optimizer/index.html
index 75a90a7d4e..8182fcf2b3 100644
--- a/api/delta_table/delta_table_optimizer/index.html
+++ b/api/delta_table/delta_table_optimizer/index.html
@@ -1347,6 +1347,8 @@
         
       
         
+      
+        
       
         
       
@@ -1410,6 +1412,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Daft
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
@@ -1711,7 +1733,7 @@ 
    
 
 
 
    
-
    TableOptimizer(table: DeltaTable)
+
    TableOptimizer(table: DeltaTable)
 
    
 
   
    
@@ -1742,7 +1764,7 @@ 
    
 
 
 
    
-
    compact(partition_filters: Optional[FilterType] = None, target_size: Optional[int] = None, max_concurrent_tasks: Optional[int] = None, min_commit_interval: Optional[Union[int, timedelta]] = None, writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None) -> Dict[str, Any]
+
    compact(partition_filters: Optional[FilterType] = None, target_size: Optional[int] = None, max_concurrent_tasks: Optional[int] = None, min_commit_interval: Optional[Union[int, timedelta]] = None, writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None) -> Dict[str, Any]
 
    
 
   
    
@@ -1913,7 +1935,7 @@ 
    
 
 
 
    
-
    z_order(columns: Iterable[str], partition_filters: Optional[FilterType] = None, target_size: Optional[int] = None, max_concurrent_tasks: Optional[int] = None, max_spill_size: int = 20 * 1024 * 1024 * 1024, min_commit_interval: Optional[Union[int, timedelta]] = None, writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None) -> Dict[str, Any]
+
    z_order(columns: Iterable[str], partition_filters: Optional[FilterType] = None, target_size: Optional[int] = None, max_concurrent_tasks: Optional[int] = None, max_spill_size: int = 20 * 1024 * 1024 * 1024, min_commit_interval: Optional[Union[int, timedelta]] = None, writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None) -> Dict[str, Any]
 
    
 
   
    
diff --git a/api/delta_table/index.html b/api/delta_table/index.html
index 6a6c24e876..7cab48f111 100644
--- a/api/delta_table/index.html
+++ b/api/delta_table/index.html
@@ -1280,6 +1280,8 @@
         
       
         
+      
+        
       
         
       
@@ -1343,6 +1345,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Daft
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
@@ -1816,7 +1838,7 @@ 
    
   
 
 
    
-
    DeltaTable(table_uri: Union[str, Path, os.PathLike[str]], version: Optional[int] = None, storage_options: Optional[Dict[str, str]] = None, without_files: bool = False, log_buffer_size: Optional[int] = None)
+
    DeltaTable(table_uri: Union[str, Path, os.PathLike[str]], version: Optional[int] = None, storage_options: Optional[Dict[str, str]] = None, without_files: bool = False, log_buffer_size: Optional[int] = None)
 
    
 
   
    
@@ -2037,7 +2059,7 @@ 
    
 
 
 
    
-
    cleanup_metadata() -> None
+
    cleanup_metadata() -> None
 
    
 
   
    
@@ -2063,7 +2085,7 @@ 
    
   
 
 
    
-
    create(table_uri: Union[str, Path], schema: Union[pyarrow.Schema, DeltaSchema], mode: Literal['error', 'append', 'overwrite', 'ignore'] = 'error', partition_by: Optional[Union[List[str], str]] = None, name: Optional[str] = None, description: Optional[str] = None, configuration: Optional[Mapping[str, Optional[str]]] = None, storage_options: Optional[Dict[str, str]] = None, custom_metadata: Optional[Dict[str, str]] = None) -> DeltaTable
+
    create(table_uri: Union[str, Path], schema: Union[pyarrow.Schema, DeltaSchema], mode: Literal['error', 'append', 'overwrite', 'ignore'] = 'error', partition_by: Optional[Union[List[str], str]] = None, name: Optional[str] = None, description: Optional[str] = None, configuration: Optional[Mapping[str, Optional[str]]] = None, storage_options: Optional[Dict[str, str]] = None, custom_metadata: Optional[Dict[str, str]] = None) -> DeltaTable
 
    
 
   
    
@@ -2269,7 +2291,7 @@ 
    
 
 
 
    
-
    delete(predicate: Optional[str] = None, writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None) -> Dict[str, Any]
+
    delete(predicate: Optional[str] = None, writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None) -> Dict[str, Any]
 
    
 
   
    
@@ -2376,7 +2398,7 @@ 
    
 
 
 
    
-
    file_uris(partition_filters: Optional[List[Tuple[str, str, Any]]] = None) -> List[str]
+
    file_uris(partition_filters: Optional[List[Tuple[str, str, Any]]] = None) -> List[str]
 
    
 
   
    
@@ -2471,7 +2493,7 @@ 
    
 
 
 
    
-
    files(partition_filters: Optional[List[Tuple[str, str, Any]]] = None) -> List[str]
+
    files(partition_filters: Optional[List[Tuple[str, str, Any]]] = None) -> List[str]
 
    
 
   
    
@@ -2569,7 +2591,7 @@ 
    
   
 
 
    
-
    from_data_catalog(data_catalog: DataCatalog, database_name: str, table_name: str, data_catalog_id: Optional[str] = None, version: Optional[int] = None, log_buffer_size: Optional[int] = None) -> DeltaTable
+
    from_data_catalog(data_catalog: DataCatalog, database_name: str, table_name: str, data_catalog_id: Optional[str] = None, version: Optional[int] = None, log_buffer_size: Optional[int] = None) -> DeltaTable
 
    
 
   
    
@@ -2694,7 +2716,7 @@ 
    
 
 
 
    
-
    get_add_actions(flatten: bool = False) -> pyarrow.RecordBatch
+
    get_add_actions(flatten: bool = False) -> pyarrow.RecordBatch
 
    
 
   
    
@@ -2795,7 +2817,7 @@ 
    
 
 
 
    
-
    history(limit: Optional[int] = None) -> List[Dict[str, Any]]
+
    history(limit: Optional[int] = None) -> List[Dict[str, Any]]
 
    
 
   
    
@@ -2871,7 +2893,7 @@ 
    
 
 
 
    
-
    load_as_version(version: Union[int, str, datetime]) -> None
+
    load_as_version(version: Union[int, str, datetime]) -> None
 
    
 
   
    
@@ -2938,7 +2960,7 @@ 
    
 
 
 
    
-
    load_version(version: int) -> None
+
    load_version(version: int) -> None
 
    
 
   
    
@@ -2993,7 +3015,7 @@ 
    
 
 
 
    
-
    load_with_datetime(datetime_string: str) -> None
+
    load_with_datetime(datetime_string: str) -> None
 
    
 
   
    
@@ -3056,7 +3078,7 @@ 
    
 
 
 
    
-
    merge(source: Union[pyarrow.Table, pyarrow.RecordBatch, pyarrow.RecordBatchReader, ds.Dataset, pd.DataFrame], predicate: str, source_alias: Optional[str] = None, target_alias: Optional[str] = None, error_on_type_mismatch: bool = True, writer_properties: Optional[WriterProperties] = None, large_dtypes: bool = True, custom_metadata: Optional[Dict[str, str]] = None) -> TableMerger
+
    merge(source: Union[pyarrow.Table, pyarrow.RecordBatch, pyarrow.RecordBatchReader, ds.Dataset, pd.DataFrame], predicate: str, source_alias: Optional[str] = None, target_alias: Optional[str] = None, error_on_type_mismatch: bool = True, writer_properties: Optional[WriterProperties] = None, large_dtypes: bool = True, custom_metadata: Optional[Dict[str, str]] = None) -> TableMerger
 
    
 
   
    
@@ -3231,7 +3253,7 @@ 
    
 
 
 
    
-
    metadata() -> Metadata
+
    metadata() -> Metadata
 
    
 
   
    
@@ -3276,7 +3298,7 @@ 
    
 
 
 
    
-
    protocol() -> ProtocolVersions
+
    protocol() -> ProtocolVersions
 
    
 
   
    
@@ -3321,7 +3343,7 @@ 
    
 
 
 
    
-
    repair(dry_run: bool = False, custom_metadata: Optional[Dict[str, str]] = None) -> Dict[str, Any]
+
    repair(dry_run: bool = False, custom_metadata: Optional[Dict[str, str]] = None) -> Dict[str, Any]
 
    
 
   
    
@@ -3402,7 +3424,7 @@ 
    
 
 
 
    
-
    restore(target: Union[int, datetime, str], *, ignore_missing_files: bool = False, protocol_downgrade_allowed: bool = False, custom_metadata: Optional[Dict[str, str]] = None) -> Dict[str, Any]
+
    restore(target: Union[int, datetime, str], *, ignore_missing_files: bool = False, protocol_downgrade_allowed: bool = False, custom_metadata: Optional[Dict[str, str]] = None) -> Dict[str, Any]
 
    
 
   
    
@@ -3519,7 +3541,7 @@ 
    
 
 
 
    
-
    schema() -> DeltaSchema
+
    schema() -> DeltaSchema
 
    
 
   
    
@@ -3564,7 +3586,7 @@ 
    
 
 
 
    
-
    to_pandas(partitions: Optional[List[Tuple[str, str, Any]]] = None, columns: Optional[List[str]] = None, filesystem: Optional[Union[str, pa_fs.FileSystem]] = None, filters: Optional[FilterType] = None) -> pd.DataFrame
+
    to_pandas(partitions: Optional[List[Tuple[str, str, Any]]] = None, columns: Optional[List[str]] = None, filesystem: Optional[Union[str, pa_fs.FileSystem]] = None, filters: Optional[FilterType] = None) -> pd.DataFrame
 
    
 
   
    
@@ -3657,7 +3679,7 @@ 
    
 
 
 
    
-
    to_pyarrow_dataset(partitions: Optional[List[Tuple[str, str, Any]]] = None, filesystem: Optional[Union[str, pa_fs.FileSystem]] = None, parquet_read_options: Optional[ParquetReadOptions] = None) -> pyarrow.dataset.Dataset
+
    to_pyarrow_dataset(partitions: Optional[List[Tuple[str, str, Any]]] = None, filesystem: Optional[Union[str, pa_fs.FileSystem]] = None, parquet_read_options: Optional[ParquetReadOptions] = None) -> pyarrow.dataset.Dataset
 
    
 
   
    
@@ -3777,7 +3799,7 @@ 
    
 
 
 
    
-
    to_pyarrow_table(partitions: Optional[List[Tuple[str, str, Any]]] = None, columns: Optional[List[str]] = None, filesystem: Optional[Union[str, pa_fs.FileSystem]] = None, filters: Optional[FilterType] = None) -> pyarrow.Table
+
    to_pyarrow_table(partitions: Optional[List[Tuple[str, str, Any]]] = None, columns: Optional[List[str]] = None, filesystem: Optional[Union[str, pa_fs.FileSystem]] = None, filters: Optional[FilterType] = None) -> pyarrow.Table
 
    
 
   
    
@@ -3870,7 +3892,7 @@ 
    
 
 
 
    
-
    update(updates: Optional[Dict[str, str]] = None, new_values: Optional[Dict[str, Union[int, float, str, datetime, bool, List[Any]]]] = None, predicate: Optional[str] = None, writer_properties: Optional[WriterProperties] = None, error_on_type_mismatch: bool = True, custom_metadata: Optional[Dict[str, str]] = None) -> Dict[str, Any]
+
    update(updates: Optional[Dict[str, str]] = None, new_values: Optional[Dict[str, Union[int, float, str, datetime, bool, List[Any]]]] = None, predicate: Optional[str] = None, writer_properties: Optional[WriterProperties] = None, error_on_type_mismatch: bool = True, custom_metadata: Optional[Dict[str, str]] = None) -> Dict[str, Any]
 
    
 
   
    
@@ -4025,7 +4047,7 @@ 
    
 
 
 
    
-
    update_incremental() -> None
+
    update_incremental() -> None
 
    
 
   
    
@@ -4047,7 +4069,7 @@ 
    
 
 
 
    
-
    vacuum(retention_hours: Optional[int] = None, dry_run: bool = True, enforce_retention_duration: bool = True, custom_metadata: Optional[Dict[str, str]] = None) -> List[str]
+
    vacuum(retention_hours: Optional[int] = None, dry_run: bool = True, enforce_retention_duration: bool = True, custom_metadata: Optional[Dict[str, str]] = None) -> List[str]
 
    
 
   
    
@@ -4142,7 +4164,7 @@ 
    
 
 
 
    
-
    version() -> int
+
    version() -> int
 
    
 
   
    
diff --git a/api/delta_table/metadata/index.html b/api/delta_table/metadata/index.html
index a2fbbe2138..bac7cdd4d0 100644
--- a/api/delta_table/metadata/index.html
+++ b/api/delta_table/metadata/index.html
@@ -1375,6 +1375,8 @@
         
       
         
+      
+        
       
         
       
@@ -1438,6 +1440,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Daft
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
@@ -1771,7 +1793,7 @@ 
    
   
 
 
    
-
    Metadata(table: RawDeltaTable)
+
    Metadata(table: RawDeltaTable)
 
    
 
   
    
diff --git a/api/delta_writer/index.html b/api/delta_writer/index.html
index 054295fe52..113305c623 100644
--- a/api/delta_writer/index.html
+++ b/api/delta_writer/index.html
@@ -1367,6 +1367,8 @@
         
       
         
+      
+        
       
         
       
@@ -1430,6 +1432,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Daft
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
@@ -1753,7 +1775,7 @@ 
    
 
 
 
    
-
    write_deltalake(table_or_uri: Union[str, Path, DeltaTable], data: Union[pd.DataFrame, ds.Dataset, pa.Table, pa.RecordBatch, Iterable[pa.RecordBatch], RecordBatchReader], *, schema: Optional[Union[pa.Schema, DeltaSchema]] = None, partition_by: Optional[Union[List[str], str]] = None, mode: Literal['error', 'append', 'overwrite', 'ignore'] = 'error', file_options: Optional[ds.ParquetFileWriteOptions] = None, max_partitions: Optional[int] = None, max_open_files: int = 1024, max_rows_per_file: int = 10 * 1024 * 1024, min_rows_per_group: int = 64 * 1024, max_rows_per_group: int = 128 * 1024, name: Optional[str] = None, description: Optional[str] = None, configuration: Optional[Mapping[str, Optional[str]]] = None, overwrite_schema: bool = False, schema_mode: Optional[Literal['merge', 'overwrite']] = None, storage_options: Optional[Dict[str, str]] = None, partition_filters: Optional[List[Tuple[str, str, Any]]] = None, predicate: Optional[str] = None, large_dtypes: bool = False, engine: Literal['pyarrow', 'rust'] = 'pyarrow', writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None) -> None
+
    write_deltalake(table_or_uri: Union[str, Path, DeltaTable], data: Union[pd.DataFrame, ds.Dataset, pa.Table, pa.RecordBatch, Iterable[pa.RecordBatch], RecordBatchReader], *, schema: Optional[Union[pa.Schema, DeltaSchema]] = None, partition_by: Optional[Union[List[str], str]] = None, mode: Literal['error', 'append', 'overwrite', 'ignore'] = 'error', file_options: Optional[ds.ParquetFileWriteOptions] = None, max_partitions: Optional[int] = None, max_open_files: int = 1024, max_rows_per_file: int = 10 * 1024 * 1024, min_rows_per_group: int = 64 * 1024, max_rows_per_group: int = 128 * 1024, name: Optional[str] = None, description: Optional[str] = None, configuration: Optional[Mapping[str, Optional[str]]] = None, overwrite_schema: bool = False, schema_mode: Optional[Literal['merge', 'overwrite']] = None, storage_options: Optional[Dict[str, str]] = None, partition_filters: Optional[List[Tuple[str, str, Any]]] = None, predicate: Optional[str] = None, large_dtypes: bool = False, engine: Literal['pyarrow', 'rust'] = 'pyarrow', writer_properties: Optional[WriterProperties] = None, custom_metadata: Optional[Dict[str, str]] = None) -> None
 
    
 
   
    
@@ -2128,7 +2150,7 @@ 
    
   
 
 
    
-
    WriterProperties(data_page_size_limit: Optional[int] = None, dictionary_page_size_limit: Optional[int] = None, data_page_row_count_limit: Optional[int] = None, write_batch_size: Optional[int] = None, max_row_group_size: Optional[int] = None, compression: Optional[Literal['UNCOMPRESSED', 'SNAPPY', 'GZIP', 'BROTLI', 'LZ4', 'ZSTD', 'LZ4_RAW']] = None, compression_level: Optional[int] = None)
+
    WriterProperties(data_page_size_limit: Optional[int] = None, dictionary_page_size_limit: Optional[int] = None, data_page_row_count_limit: Optional[int] = None, write_batch_size: Optional[int] = None, max_row_group_size: Optional[int] = None, compression: Optional[Literal['UNCOMPRESSED', 'SNAPPY', 'GZIP', 'BROTLI', 'LZ4', 'ZSTD', 'LZ4_RAW']] = None, compression_level: Optional[int] = None)
 
    
 
   
    
@@ -2288,7 +2310,7 @@ 
    
 
 
 
    
-
    convert_to_deltalake(uri: Union[str, Path], mode: Literal['error', 'ignore'] = 'error', partition_by: Optional[pa.Schema] = None, partition_strategy: Optional[Literal['hive']] = None, name: Optional[str] = None, description: Optional[str] = None, configuration: Optional[Mapping[str, Optional[str]]] = None, storage_options: Optional[Dict[str, str]] = None, custom_metadata: Optional[Dict[str, str]] = None) -> None
+
    convert_to_deltalake(uri: Union[str, Path], mode: Literal['error', 'ignore'] = 'error', partition_by: Optional[pa.Schema] = None, partition_strategy: Optional[Literal['hive']] = None, name: Optional[str] = None, description: Optional[str] = None, configuration: Optional[Mapping[str, Optional[str]]] = None, storage_options: Optional[Dict[str, str]] = None, custom_metadata: Optional[Dict[str, str]] = None) -> None
 
    
 
   
    
diff --git a/api/exceptions/index.html b/api/exceptions/index.html
index fbf7573431..a47f28972f 100644
--- a/api/exceptions/index.html
+++ b/api/exceptions/index.html
@@ -1348,6 +1348,8 @@
         
       
         
+      
+        
       
         
       
@@ -1411,6 +1413,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Daft
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/api/schema/index.html b/api/schema/index.html
index 977c374558..39400dc798 100644
--- a/api/schema/index.html
+++ b/api/schema/index.html
@@ -1686,6 +1686,8 @@
         
       
         
+      
+        
       
         
       
@@ -1749,6 +1751,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Daft
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
@@ -2394,7 +2416,7 @@ 
    
 
 
 
    
-
    Schema(fields: List[Field])
+
    Schema(fields: List[Field])
 
    
 
   
    
@@ -2469,7 +2491,7 @@ 
    
   
 
 
    
-
    from_json(schema_json) -> Schema
+
    from_json(schema_json) -> Schema
 
    
 
   
    
@@ -2535,7 +2557,7 @@ 
    
   
 
 
    
-
    from_pyarrow(data_type) -> Schema
+
    from_pyarrow(data_type) -> Schema
 
    
 
   
    
@@ -2615,7 +2637,7 @@ 
    
   
 
 
    
-
    to_json() -> str
+
    to_json() -> str
 
    
 
   
    
@@ -2671,7 +2693,7 @@ 
    
   
 
 
    
-
    to_pyarrow(as_large_types: bool = False) -> pyarrow.Schema
+
    to_pyarrow(as_large_types: bool = False) -> pyarrow.Schema
 
    
 
   
    
@@ -2755,7 +2777,7 @@ 
    
 
 
 
    
-
    Field(name: str, type: DataType, *, nullable: bool = True, metadata: Optional[Dict[str, Any]] = None)
+
    Field(name: str, type: DataType, *, nullable: bool = True, metadata: Optional[Dict[str, Any]] = None)
 
    
 
   
    
@@ -2866,7 +2888,7 @@ 
    
   
 
 
    
-
    from_json(field_json) -> Field
+
    from_json(field_json) -> Field
 
    
 
   
    
@@ -2957,7 +2979,7 @@ 
    
   
 
 
    
-
    from_pyarrow(field: pyarrow.Field) -> Field
+
    from_pyarrow(field: pyarrow.Field) -> Field
 
    
 
   
    
@@ -3037,7 +3059,7 @@ 
    
   
 
 
    
-
    to_json() -> str
+
    to_json() -> str
 
    
 
   
    
@@ -3092,7 +3114,7 @@ 
    
   
 
 
    
-
    to_pyarrow() -> pyarrow.Field
+
    to_pyarrow() -> pyarrow.Field
 
    
 
   
    
@@ -3147,7 +3169,7 @@ 
    
 
 
 
    
-
    PrimitiveType(data_type: str)
+
    PrimitiveType(data_type: str)
 
    
 
   
    
@@ -3201,7 +3223,7 @@ 
    
   
 
 
    
-
    from_json(type_json) -> PrimitiveType
+
    from_json(type_json) -> PrimitiveType
 
    
 
   
    
@@ -3281,7 +3303,7 @@ 
    
   
 
 
    
-
    from_pyarrow(data_type) -> PrimitiveType
+
    from_pyarrow(data_type) -> PrimitiveType
 
    
 
   
    
@@ -3361,7 +3383,7 @@ 
    
   
 
 
    
-
    to_pyarrow() -> pyarrow.DataType
+
    to_pyarrow() -> pyarrow.DataType
 
    
 
   
    
@@ -3390,7 +3412,7 @@ 
    
 
 
 
    
-
    ArrayType(element_type: DataType, *, contains_null: bool = True)
+
    ArrayType(element_type: DataType, *, contains_null: bool = True)
 
    
 
   
    
@@ -3482,7 +3504,7 @@ 
    
   
 
 
    
-
    from_json(type_json) -> ArrayType
+
    from_json(type_json) -> ArrayType
 
    
 
   
    
@@ -3575,7 +3597,7 @@ 
    
   
 
 
    
-
    from_pyarrow(data_type) -> ArrayType
+
    from_pyarrow(data_type) -> ArrayType
 
    
 
   
    
@@ -3655,7 +3677,7 @@ 
    
   
 
 
    
-
    to_json() -> str
+
    to_json() -> str
 
    
 
   
    
@@ -3680,7 +3702,7 @@ 
    
   
 
 
    
-
    to_pyarrow() -> pyarrow.ListType
+
    to_pyarrow() -> pyarrow.ListType
 
    
 
   
    
@@ -3709,7 +3731,7 @@ 
    
 
 
 
    
-
    MapType(key_type: DataType, value_type: DataType, *, value_contains_null: bool = True)
+
    MapType(key_type: DataType, value_type: DataType, *, value_contains_null: bool = True)
 
    
 
   
    
@@ -3820,7 +3842,7 @@ 
    
   
 
 
    
-
    from_json(type_json) -> MapType
+
    from_json(type_json) -> MapType
 
    
 
   
    
@@ -3914,7 +3936,7 @@ 
    
   
 
 
    
-
    from_pyarrow(data_type) -> MapType
+
    from_pyarrow(data_type) -> MapType
 
    
 
   
    
@@ -3994,7 +4016,7 @@ 
    
   
 
 
    
-
    to_json() -> str
+
    to_json() -> str
 
    
 
   
    
@@ -4043,7 +4065,7 @@ 
    
   
 
 
    
-
    to_pyarrow() -> pyarrow.MapType
+
    to_pyarrow() -> pyarrow.MapType
 
    
 
   
    
@@ -4072,7 +4094,7 @@ 
    
 
 
 
    
-
    StructType(fields: List[Field])
+
    StructType(fields: List[Field])
 
    
 
   
    
@@ -4145,7 +4167,7 @@ 
    
   
 
 
    
-
    from_json(type_json) -> StructType
+
    from_json(type_json) -> StructType
 
    
 
   
    
@@ -4235,7 +4257,7 @@ 
    
   
 
 
    
-
    from_pyarrow(data_type) -> StructType
+
    from_pyarrow(data_type) -> StructType
 
    
 
   
    
@@ -4315,7 +4337,7 @@ 
    
   
 
 
    
-
    to_json() -> str
+
    to_json() -> str
 
    
 
   
    
@@ -4370,7 +4392,7 @@ 
    
   
 
 
    
-
    to_pyarrow() -> pyarrow.StructType
+
    to_pyarrow() -> pyarrow.StructType
 
    
 
   
    
diff --git a/api/storage/index.html b/api/storage/index.html
index 8ac776b060..5fb163c292 100644
--- a/api/storage/index.html
+++ b/api/storage/index.html
@@ -1361,6 +1361,8 @@
         
       
         
+      
+        
       
         
       
@@ -1424,6 +1426,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Daft
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
@@ -1740,7 +1762,7 @@ 
    
 
 
 
    
-
    DeltaStorageHandler(root: str, options: dict[str, str] | None = None, known_sizes: dict[str, int] | None = None)
+
    DeltaStorageHandler(root: str, options: dict[str, str] | None = None, known_sizes: dict[str, int] | None = None)
 
    
 
   
    
@@ -1773,7 +1795,7 @@ 
    get_file_info_selector(selector: FileSelector) -> List[FileInfo]
+
    get_file_info_selector(selector: FileSelector) -> List[FileInfo]
 
    
 
   
    
@@ -1848,7 +1870,7 @@ 
    open_input_file(path: str) -> pa.PythonFile
+
    open_input_file(path: str) -> pa.PythonFile
 
    
 
   
    
@@ -1923,7 +1945,7 @@ 
    open_input_stream(path: str) -> pa.PythonFile
+
    open_input_stream(path: str) -> pa.PythonFile
 
    
 
   
    
@@ -1998,7 +2020,7 @@ 
    open_output_stream(path: str, metadata: Optional[Dict[str, str]] = None) -> pa.PythonFile
+
    open_output_stream(path: str, metadata: Optional[Dict[str, str]] = None) -> pa.PythonFile
 
    
 
   
    
diff --git a/assets/_mkdocstrings.css b/assets/_mkdocstrings.css
index 4b7d98b833..57a23e10f5 100644
--- a/assets/_mkdocstrings.css
+++ b/assets/_mkdocstrings.css
@@ -106,4 +106,9 @@ code.doc-symbol-module {
 
 code.doc-symbol-module::after {
   content: "mod";
-}
\ No newline at end of file
+}
+
+.doc-signature .autorefs {
+  color: inherit;
+  border-bottom: 1px dotted currentcolor;
+}
diff --git a/delta-lake-best-practices/index.html b/delta-lake-best-practices/index.html
index 2bb629ba94..de27ccf812 100644
--- a/delta-lake-best-practices/index.html
+++ b/delta-lake-best-practices/index.html
@@ -1397,6 +1397,8 @@
         
       
         
+      
+        
       
         
       
@@ -1460,6 +1462,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Daft
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/delta-lake-big-data-small-data/index.html b/delta-lake-big-data-small-data/index.html
index 29f83e235c..3b7ce5755d 100644
--- a/delta-lake-big-data-small-data/index.html
+++ b/delta-lake-big-data-small-data/index.html
@@ -1355,6 +1355,8 @@
         
       
         
+      
+        
       
         
       
@@ -1418,6 +1420,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Daft
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/how-delta-lake-works/architecture-of-delta-table/index.html b/how-delta-lake-works/architecture-of-delta-table/index.html
index cad00fd4b2..de9d3fa62d 100644
--- a/how-delta-lake-works/architecture-of-delta-table/index.html
+++ b/how-delta-lake-works/architecture-of-delta-table/index.html
@@ -1280,6 +1280,8 @@
         
       
         
+      
+        
       
         
       
@@ -1343,6 +1345,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Daft
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/how-delta-lake-works/delta-lake-acid-transactions/index.html b/how-delta-lake-works/delta-lake-acid-transactions/index.html
index 41b9eca812..4d899c04fe 100644
--- a/how-delta-lake-works/delta-lake-acid-transactions/index.html
+++ b/how-delta-lake-works/delta-lake-acid-transactions/index.html
@@ -1280,6 +1280,8 @@
         
       
         
+      
+        
       
         
       
@@ -1343,6 +1345,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Daft
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/how-delta-lake-works/delta-lake-file-skipping/index.html b/how-delta-lake-works/delta-lake-file-skipping/index.html
index 24493431dd..afaa76b942 100644
--- a/how-delta-lake-works/delta-lake-file-skipping/index.html
+++ b/how-delta-lake-works/delta-lake-file-skipping/index.html
@@ -1278,6 +1278,8 @@
         
       
         
+      
+        
       
         
       
@@ -1341,6 +1343,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Daft
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/index.html b/index.html
index 99c6f808a3..dfd2242156 100644
--- a/index.html
+++ b/index.html
@@ -1278,6 +1278,8 @@
         
       
         
+      
+        
       
         
       
@@ -1341,6 +1343,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Daft
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
diff --git a/integrations/delta-lake-arrow/index.html b/integrations/delta-lake-arrow/index.html
index 08271eef76..a4313b43ed 100644
--- a/integrations/delta-lake-arrow/index.html
+++ b/integrations/delta-lake-arrow/index.html
@@ -14,7 +14,7 @@
         
       
       
-        
+        
       
       
       
@@ -1282,6 +1282,8 @@
         
       
         
+      
+        
       
         
       
@@ -1418,6 +1420,26 @@
   
   
   
+    
  • 
+      
+        
+  
+  
+    Daft
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
     
  • 
       
         
@@ -1828,13 +1850,13 @@ 
    Conclusion
    
         
         
           
-              
+          
             
    
               
                 Next
               
               
    
-                Dagster
+                Daft
               
    
             
    
             
    
diff --git a/integrations/delta-lake-daft/index.html b/integrations/delta-lake-daft/index.html
new file mode 100644
index 0000000000..ebfbe27f6d
--- /dev/null
+++ b/integrations/delta-lake-daft/index.html
@@ -0,0 +1,2156 @@
+
+
+
+  
+    
+      
+      
+      
+      
+      
+        
+      
+      
+        
+      
+      
+        
+      
+      
+      
+      
+    
+    
+      
+        Daft - Delta Lake Documentation
+      
+    
+    
+      
+      
+        
+        
+      
+      
+
+
+    
+    
+      
+    
+    
+      
+        
+        
+        
+        
+        
+      
+    
+    
+      
+    
+      
+    
+    
+    
+      
+
+    
+    
+    
+  
+  
+  
+    
+    
+      
+    
+    
+    
+    
+    
+  
+    
+    
+      
+    
+    
+    
+    
+    
+    
    
+      
+    
    
+    
+    
+      
+
+  
+
+
    
+  
+  
+    
+      
+
+    
+  
+
    
+    
+    
    
+      
+      
+        
+      
+      
    
+        
    
+          
+            
+              
+              
    
+                
    
+                  
    
+                    
+
+
+  
+
+
+
+                  
    
+                
    
+              
    
+            
+            
+              
+              
    
+                
    
+                  
    
+                    
+
+
+                  
    
+                
    
+              
    
+            
+          
+          
+            
    
+              
    
+                
+                  
+
+  
+  
+
+
+
    Using Delta Lake with Daft
    
+
    Daft is a framework for ETL, analytics, and ML/AI at scale with a familiar Python dataframe API, implemented in Rust.
    
+
    For Delta Lake users, Daft is a great data processing tool because it gives you the following features:
    
+
      
+
    • Skipping Filtered Data: Daft implements automatic partition pruning and stats-based file pruning for filter predicates, skipping data that doesn’t need to be read.
      • 
+
    • Multimodal Dataframes: read, write and transform multimodal data incl. images, JSON, PDF, audio, etc.
      • 
+
    • Parallel + Distributed Reads: Daft parallelizes Delta Lake table reads over all cores of your machine, if using the default multithreading runner, or all cores + machines of your Ray cluster, if using the distributed Ray runner.
      • 
+
    • Multi-cloud Support: Daft supports reading Delta Lake tables from AWS S3, Azure Blob Store, and GCS, as well as local files.
      • 
+
    
+
    Daft and Delta Lake work really well together. Daft provides unified compute for Delta Lake’s unified storage. Together, Delta Lake and Daft give you high-performance query optimization and distributed compute on massive datasets.
    
+
    Installing Daft for Delta Lake
    
+
    The easiest way to use Delta Lake format with Daft DataFrames is to install Daft with the [deltalake] extras using pip:
    
+
    !pip install -U "getdaft[deltalake]"
+
    
+
    This adds the deltalake Python package to your install. This package is used to fetch metadata about the Delta Lake table, such as paths to the underlying Parquet files and table statistics. You can of course also install the deltalake manually.
    
+
    Read Delta Lake into a Daft DataFrame
    
+
    You can easily read Delta Lake tables into a Daft DataFrame using the read_delta_lake method. Let's use it to read some data stored in a Delta Lake on disk. You can access the data stored as a Delta Lake on Github
    
+
    import daft
+
+# read delta table into Daft DataFrame
+df = daft.read_delta_lake("path/to/delta_table")
+
    
+
    You can also read in Delta Lake data from remote sources like S3:
    
+
    # table_uri = (
+#     "s3://daft-public-datasets/red-pajamas/"
+#     "stackexchange-sample-north-germanic-deltalake"
+# )
+# df = daft.read_delta_lake(table_uri)
+
    
+
    df
+
    
+
    
+
+
+
    first_name
    Utf8    		last_name
    Utf8    		country
    Utf8    		continent
    Utf8
    
+(No data to display: Dataframe not materialized)
+
    
+
+
    Daft DataFrames are lazy by default. This means that the contents will not be computed ("materialized") unless you explicitly tell Daft to do so. This is best practice for working with larger-than-memory datasets and parallel/distributed architectures.
    
+
    The Delta table we have just loaded only has 5 rows. You can materialize it in memory using the .collect method:
    
+
    > df.collect()
+
+|    | first_name   | last_name   | country   | continent   |
+|---:|:-------------|:------------|:----------|:------------|
+|  0 | Ernesto      | Guevara     | Argentina | NaN         |
+|  1 | Bruce        | Lee         | China     | Asia        |
+|  2 | Jack         | Ma          | China     | Asia        |
+|  3 | Wolfgang     | Manche      | Germany   | NaN         |
+|  4 | Soraya       | Jala        | Germany   | NaN         |
+
    
+
    What can I do with a Daft DataFrame?
    
+
    Daft gives you full-featured DataFrame functionality, similar to what you might be used to from pandas, Dask or PySpark.
    
+
    On top of this, Daft also gives you:
    
+
      
+
    • Multimodal data type support to work with Images, URLs, Tensors and more
      • 
+
    • Expressions API for easy column transformations
      • 
+
    • UDFs for multi-column transformation, incl. ML applications
      • 
+
    
+
    Let's take a quick look at some of Daft's basic DataFrame operations.
    
+
    You can select columns from your DataFrame using the select method. We'll use the show method to show the first n rows (defaults to 10):
    
+
    > df.select("first_name", "country").show()
+
+|    | first_name   | country   |
+|---:|:-------------|:----------|
+|  0 | Ernesto      | Argentina |
+|  1 | Bruce        | China     |
+|  2 | Jack         | China     |
+|  3 | Wolfgang     | Germany   |
+|  4 | Soraya       | Germany   |
+
    
+
    You can sort your Daft DataFrame using the sort method:
    
+
    > df.sort(df["country"], desc=True).show()
+
+|    | first_name   | last_name   | country   | continent   |
+|---:|:-------------|:------------|:----------|:------------|
+|  0 | Wolfgang     | Manche      | Germany   | NaN         |
+|  1 | Soraya       | Jala        | Germany   | NaN         |
+|  2 | Bruce        | Lee         | China     | Asia        |
+|  3 | Jack         | Ma          | China     | Asia        |
+|  4 | Ernesto      | Guevara     | Argentina | NaN         |
+
    
+
    You can filter your DataFrame using the where method:
    
+
    > df.where(df["continent"] == "Asia").show()
+
+|    | first_name   | last_name   | country   | continent   |
+|---:|:-------------|:------------|:----------|:------------|
+|  0 | Bruce        | Lee         | China     | Asia        |
+|  1 | Jack         | Ma          | China     | Asia        |
+
    
+
    You can group your DataFrame by a specific columns using the groupby method. You can then specify the aggregation method, in this case using the count aggregator:
    
+
    > df.select("first_name", "country").groupby(df["country"]).count("first_name").show()
+
+|    | country   |   first_name |
+|---:|:----------|-------------:|
+|  0 | Germany   |            2 |
+|  1 | China     |            2 |
+|  2 | Argentina |            1 |
+
    
+
    Check out the Daft User Guide for a complete list of DataFrame operations.
    
+
    Data Skipping Optimizations
    
+
    You may have noticed the Delta Lake warning at the top when we first called collect on our DataFrame:
    
+
    WARNING: has partitioning keys = [PartitionField(country#Utf8)], but no partition filter was specified. This will result in a full table scan.
    
+
+
    Delta Lake is informing us that the data is partitioned on the country column.
    
+
    Daft's native query optimizer has access to all of the Delta Lake metadata.
    
+
    This means it can optimize your query by skipping the partitions that are not relevant for this query. Instead of having to read all 3 partitions, we can read only 1 and get the same result, just faster!
    
+
    # Filter on partition columns will result in efficient partition pruning; non-matching partitions will be skipped.
+> df.where(df["country"] == "Germany").show()
+
+|    | first_name   | last_name   | country   |   continent |
+|---:|:-------------|:------------|:----------|------------:|
+|  0 | Wolfgang     | Manche      | Germany   |         nan |
+|  1 | Soraya       | Jala        | Germany   |         nan |
+
    
+
    You can use the explain method to see how Daft is optimizing your query. Since we've already called collect on our DataFrame, it is already in memory. So below we copy the output of explain(show_all=True) before calling collect:
    
+
    Running df.where(df["continent"] == "Asia").explain(True) returns:
    
+
    (...)
+
+== Optimized Logical Plan ==
+
+* PythonScanOperator: DeltaLakeScanOperator(None)
+|   File schema = first_name#Utf8, last_name#Utf8, country#Utf8, continent#Utf8
+|   Partitioning keys = [PartitionField(country#Utf8)]
+|   Filter pushdown = col(continent) == lit("Asia")
+|   Output schema = first_name#Utf8, last_name#Utf8, country#Utf8, continent#Utf8
+
+
+== Physical Plan ==
+
+* TabularScan:
+|   Num Scan Tasks = 3
+|   Estimated Scan Bytes = 3045
+|   Clustering spec = { Num partitions = 3 }
+
    
+
    Whereas running df.where(df["country"] == "Germany").explain(True) returns:
    
+
    (...)
+
+== Optimized Logical Plan ==
+
+* PythonScanOperator: DeltaLakeScanOperator(None)
+|   File schema = first_name#Utf8, last_name#Utf8, country#Utf8, continent#Utf8
+|   Partitioning keys = [PartitionField(country#Utf8)]
+|   Partition Filter = col(country) == lit("Germany")
+|   Output schema = first_name#Utf8, last_name#Utf8, country#Utf8, continent#Utf8
+
+
+== Physical Plan ==
+
+* TabularScan:
+|   Num Scan Tasks = 1
+|   Estimated Scan Bytes = 1025
+|   Clustering spec = { Num partitions = 1 }
+
    
+
    Running a query on a non-partitioned column like continent will require reading in all partitions, totalling 3045 bytes in this case.
    
+
    Instead, running a query on a partitioned column (country in this case) means Daft only has to read only the relevant partition, saving us a whopping 2000+ bytes in this toy example :)
    
+
    You can read High-Performance Querying on Massive Delta Lake Tables with Daft for an in-depth benchmarking of query optimization with Delta Lake and Daft.
    
+
    Transform columns with Expressions
    
+
    Daft provides a flexible Expressions API for defining computation that needs to happen over your columns.
    
+
    For example, we can use daft.col() expressions together with the with_column method to create a new column full_name, joining the contents of the last_name column to the first_name column:
    
+
    > df_full = df.with_column("full_name", daft.col('first_name') + ' ' + daft.col('last_name'))
+> df_full.show()
+
+|    | first_name   | last_name   | country   | continent   | full_name       |
+|---:|:-------------|:------------|:----------|:------------|:----------------|
+|  0 | Ernesto      | Guevara     | Argentina | NaN         | Ernesto Guevara |
+|  1 | Bruce        | Lee         | China     | Asia        | Bruce Lee       |
+|  2 | Jack         | Ma          | China     | Asia        | Jack Ma         |
+|  3 | Wolfgang     | Manche      | Germany   | NaN         | Wolfgang Manche |
+|  4 | Soraya       | Jala        | Germany   | NaN         | Soraya Jala     |
+
    
+
    Multimodal Data Type Support
    
+
    Daft has a rich multimodal type-system with support for Python objects, Images, URLs, Tensors and more.
    
+
    Daft columns can contain any Python objects. For example, let's add a column containing a Python class Dog for some of the people in our dataset:
    
+
    > import numpy as np
+
+> class Dog:
+>     def __init__(self, name):
+>         self.name = name
+
+>     def bark(self):
+>         return f"{self.name}!"
+
+> df_dogs = daft.from_pydict({
+>     'full_name': ['Ernesto Guevara','Bruce Lee','Jack Ma','Wolfgang Manche','Soraya Jala'],
+>     "dogs": [Dog("ruffles"), Dog("shnoodles"), Dog("waffles"), Dog("doofus"), Dog("Fluffles")],
+> })
+
+> df_dogs.show()
+
+|    | full_name       | dogs                                 |
+|---:|:----------------|:-------------------------------------|
+|  0 | Ernesto Guevara | <__main__.Dog object at 0x1603d1c10> |
+|  1 | Bruce Lee       | <__main__.Dog object at 0x126ab9b90> |
+|  2 | Jack Ma         | <__main__.Dog object at 0x1603d27d0> |
+|  3 | Wolfgang Manche | <__main__.Dog object at 0x1603d1cd0> |
+|  4 | Soraya Jala     | <__main__.Dog object at 0x1603d3f50> |
+
    
+
    You can join this new dogs column to your existing DataFrame using the join method:
    
+
    > df_family = df_full.join(df_dogs, on=["full_name"])
+> df_family.show()
+
+|    | full_name       | first_name   | last_name   | country   | continent   | dogs                                 |
+|---:|:----------------|:-------------|:------------|:----------|:------------|:-------------------------------------|
+|  0 | Ernesto Guevara | Ernesto      | Guevara     | Argentina | NaN         | <__main__.Dog object at 0x1603d1c10> |
+|  1 | Bruce Lee       | Bruce        | Lee         | China     | Asia        | <__main__.Dog object at 0x126ab9b90> |
+|  2 | Jack Ma         | Jack         | Ma          | China     | Asia        | <__main__.Dog object at 0x1603d27d0> |
+|  3 | Wolfgang Manche | Wolfgang     | Manche      | Germany   | NaN         | <__main__.Dog object at 0x1603d1cd0> |
+|  4 | Soraya Jala     | Soraya       | Jala        | Germany   | NaN         | <__main__.Dog object at 0x1603d3f50> |
+
    
+
    We can then use the apply method to apply a function to each instance of the Dog class:
    
+
    > from daft import DataType
+
+> df_family = df_family.with_column(
+>     "dogs_bark_name",
+>     df_family["dogs"].apply(lambda dog: dog.bark(), return_dtype=DataType.string()),
+> )
+
+> df_family.show()
+
+|    | first_name   | last_name   | country   | continent   | full_name       | dogs                                 | dogs_bark_name   |
+|---:|:-------------|:------------|:----------|:------------|:----------------|:-------------------------------------|:-----------------|
+|  0 | Ernesto      | Guevara     | Argentina | NaN         | Ernesto Guevara | <__main__.Dog object at 0x1603d1c10> | ruffles!         |
+|  1 | Bruce        | Lee         | China     | Asia        | Bruce Lee       | <__main__.Dog object at 0x126ab9b90> | shnoodles!       |
+|  2 | Jack         | Ma          | China     | Asia        | Jack Ma         | <__main__.Dog object at 0x1603d27d0> | waffles!         |
+|  3 | Wolfgang     | Manche      | Germany   | NaN         | Wolfgang Manche | <__main__.Dog object at 0x1603d1cd0> | doofus!          |
+|  4 | Soraya       | Jala        | Germany   | NaN         | Soraya Jala     | <__main__.Dog object at 0x1603d3f50> | Fluffles!        |
+
    
+
    Daft DataFrames can also contain many other data types, like tensors, JSON, URLs and images. The Expressions API provides useful tools to work with these data types.
    
+
    Take a look at the notebook in the delta-examples Github repository for a closer look at how Daft handles URLs, images and ML applications.
    
+
    Transform multiple columns with UDFs
    
+
    You can use User-Defined Functions (UDFs) to run functions over multiple rows or columns:
    
+
    > from daft import udf
+
+> @udf(return_dtype=DataType.string())
+> def custom_bark(dog_series, owner_series):
+>     return [
+>         f"{dog.name} loves {owner_name}!"
+>         for dog, owner_name
+>         in zip(dog_series.to_pylist(), owner_series.to_pylist())
+>     ]
+
+> df_family = df_family.with_column("custom_bark", custom_bark(df_family["dogs"], df_family["first_name"]))
+> df_family.select("full_name", "dogs_bark_name", "custom_bark").show()
+
+|    | full_name       | dogs_bark_name   | custom_bark            |
+|---:|:----------------|:-----------------|:-----------------------|
+|  0 | Ernesto Guevara | ruffles!         | ruffles loves Ernesto! |
+|  1 | Bruce Lee       | shnoodles!       | shnoodles loves Bruce! |
+|  2 | Jack Ma         | waffles!         | waffles loves Jack!    |
+|  3 | Wolfgang Manche | doofus!          | doofus loves Wolfgang! |
+|  4 | Soraya Jala     | Fluffles!        | Fluffles loves Soraya! |
+
    
+
    Daft supports workloads with many more data types than traditional DataFrame APIs.
    
+
    By combining multimodal data support with the UDF functionality you can run ML workloads right within your DataFrame.
    
+
    When should I use Daft DataFrames?
    
+
    Daft DataFrames are designed for multimodal, distributed workloads.
    
+
    You may want to consider using Daft if you're working with:
    
+
      
+
    1. Large datasets that don't fit into memory or would benefit from parallelization
      2. 
+
    2. Multimodal data types, such as images, JSON, vector embeddings, and tensors
      3. 
+
    3. ML workloads that would benefit from interactive computation within DataFrame (via UDFs)
      4. 
+
    
+
    Take a look at the Daft tutorials for in-depth examples of each use case.
    
+
    Contribute to daft
    
+
    Excited about Daft and want to contribute? Join them on Github 🚀
    
+
    Like many technologies, Daft collects some non-identifiable telemetry to improve the product. This is stricly non-identifiable metadata. You can disable telemetry by setting the following environment variable: DAFT_ANALYTICS_ENABLED=0. Read more in the Daft documentation.
    
+
+
+
+
+
+                
+              
    
+            
    
+          
+          
+  
+
+        
    
+        
+      
    
+      
+        
+      
+    
    
+    
    
+      
    
+    
    
+    
+    
+    
+    
+    
+      
+      
+    
+  
+
\ No newline at end of file
diff --git a/integrations/delta-lake-dagster/index.html b/integrations/delta-lake-dagster/index.html
index 79c0286211..78b2ac4aeb 100644
--- a/integrations/delta-lake-dagster/index.html
+++ b/integrations/delta-lake-dagster/index.html
@@ -11,7 +11,7 @@
         
       
       
-        
+        
       
       
         
@@ -1282,6 +1282,8 @@
         
       
         
+      
+        
       
         
       
@@ -1342,6 +1344,26 @@
                 
   
   
+  
+    
  • 
+      
+        
+  
+  
+    Daft
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
     
   
   
@@ -2085,7 +2107,7 @@ 
    Contribute