From a97d7890ba20745895d4f1a187f640db2fe594c6 Mon Sep 17 00:00:00 2001 From: Github Action Date: Wed, 18 Oct 2023 16:31:50 +0000 Subject: [PATCH] doc update for tag --- python/_sources/usage.rst.txt | 153 ++++++++--- python/api_reference.html | 474 +++++++++++++++++++++++++++++----- python/genindex.html | 40 ++- python/index.html | 3 +- python/installation.html | 2 +- python/objects.inv | Bin 975 -> 1112 bytes python/py-modindex.html | 2 +- python/search.html | 2 +- python/searchindex.js | 2 +- python/usage.html | 84 +++++- 10 files changed, 649 insertions(+), 113 deletions(-) diff --git a/python/_sources/usage.rst.txt b/python/_sources/usage.rst.txt index 5dc5a0959e..569546dce8 100644 --- a/python/_sources/usage.rst.txt +++ b/python/_sources/usage.rst.txt @@ -14,8 +14,8 @@ of the table, and other metadata such as creation time. >>> dt.version() 3 >>> dt.files() - ['part-00000-cb6b150b-30b8-4662-ad28-ff32ddab96d2-c000.snappy.parquet', - 'part-00000-7c2deba3-1994-4fb8-bc07-d46c948aa415-c000.snappy.parquet', + ['part-00000-cb6b150b-30b8-4662-ad28-ff32ddab96d2-c000.snappy.parquet', + 'part-00000-7c2deba3-1994-4fb8-bc07-d46c948aa415-c000.snappy.parquet', 'part-00001-c373a5bd-85f0-4758-815e-7eb62007a15c-c000.snappy.parquet'] @@ -54,7 +54,7 @@ being used. We try to support many of the well-known formats to identify basic s * gs:/// -Alternatively, if you have a data catalog you can load it by reference to a +Alternatively, if you have a data catalog you can load it by reference to a database and table name. Currently supported are AWS Glue and Databricks Unity Catalog. For AWS Glue catalog, use AWS environment variables to authenticate. @@ -76,12 +76,15 @@ For Databricks Unity Catalog authentication, use environment variables: .. code-block:: python - >>> from deltalake import DataCatalog, DeltaTable - >>> catalog_name = 'main' - >>> schema_name = 'db_schema' - >>> table_name = 'db_table' - >>> data_catalog = DataCatalog.UNITY - >>> dt = DeltaTable.from_data_catalog(data_catalog=data_catalog, data_catalog_id=catalog_name, database_name=schema_name, table_name=table_name) + >>> import os + >>> from deltalake import DataCatalog, DeltaTable + >>> os.environ['DATABRICKS_WORKSPACE_URL'] = "https://adb-62800498333851.30.azuredatabricks.net" + >>> os.environ['DATABRICKS_ACCESS_TOKEN'] = "" + >>> catalog_name = 'main' + >>> schema_name = 'db_schema' + >>> table_name = 'db_table' + >>> data_catalog = DataCatalog.UNITY + >>> dt = DeltaTable.from_data_catalog(data_catalog=data_catalog, data_catalog_id=catalog_name, database_name=schema_name, table_name=table_name) .. _`s3 options`: https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html#variants .. _`azure options`: https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html#variants @@ -92,7 +95,7 @@ Custom Storage Backends While delta always needs its internal storage backend to work and be properly configured, in order to manage the delta log, it may sometime be advantageous - and is common practice in the arrow world - to customize the storage interface used for -reading the bulk data. +reading the bulk data. ``deltalake`` will work with any storage compliant with :class:`pyarrow.fs.FileSystem`, however the root of the filesystem has to be adjusted to point at the root of the Delta table. We can achieve this by wrapping the custom filesystem into @@ -102,10 +105,10 @@ a :class:`pyarrow.fs.SubTreeFileSystem`. import pyarrow.fs as fs from deltalake import DeltaTable - + path = "" filesystem = fs.SubTreeFileSystem(path, fs.LocalFileSystem()) - + dt = DeltaTable(path) ds = dt.to_pyarrow_dataset(filesystem=filesystem) @@ -176,8 +179,8 @@ Schema ~~~~~~ The schema for the table is also saved in the transaction log. It can either be -retrieved in the Delta Lake form as :class:`deltalake.schema.Schema` or as a PyArrow -schema. The first allows you to introspect any column-level metadata stored in +retrieved in the Delta Lake form as :class:`deltalake.schema.Schema` or as a PyArrow +schema. The first allows you to introspect any column-level metadata stored in the schema, while the latter represents the schema the table will be loaded into. Use :meth:`DeltaTable.schema` to retrieve the delta lake schema: @@ -209,14 +212,14 @@ History ~~~~~~~ Depending on what system wrote the table, the delta table may have provenance -information describing what operations were performed on the table, when, and +information describing what operations were performed on the table, when, and by whom. This information is retained for 30 days by default, unless otherwise specified by the table configuration ``delta.logRetentionDuration``. .. note:: - This information is not written by all writers and different writers may use - different schemas to encode the actions. For Spark's format, see: + This information is not written by all writers and different writers may use + different schemas to encode the actions. For Spark's format, see: https://docs.delta.io/latest/delta-utility.html#history-schema To view the available history, use :meth:`DeltaTable.history`: @@ -236,7 +239,7 @@ To view the available history, use :meth:`DeltaTable.history`: Current Add Actions ~~~~~~~~~~~~~~~~~~~ -The active state for a delta table is determined by the Add actions, which +The active state for a delta table is determined by the Add actions, which provide the list of files that are part of the table and metadata about them, such as creation time, size, and statistics. You can get a data frame of the add actions data using :meth:`DeltaTable.get_add_actions`: @@ -265,18 +268,18 @@ Querying Delta Tables --------------------- Delta tables can be queried in several ways. By loading as Arrow data or an Arrow -dataset, they can be used by compatible engines such as Pandas and DuckDB. By +dataset, they can be used by compatible engines such as Pandas and DuckDB. By passing on the list of files, they can be loaded into other engines such as Dask. Delta tables are often larger than can fit into memory on a single computer, so -this module provides ways to read only the parts of the data you need. Partition +this module provides ways to read only the parts of the data you need. Partition filters allow you to skip reading files that are part of irrelevant partitions. Only loading the columns required also saves memory. Finally, some methods allow reading tables batch-by-batch, allowing you to process the whole table while only having a portion loaded at any given time. To load into Pandas or a PyArrow table use the :meth:`DeltaTable.to_pandas` and -:meth:`DeltaTable.to_pyarrow_table` methods, respectively. Both of these +:meth:`DeltaTable.to_pyarrow_table` methods, respectively. Both of these support filtering partitions and selecting particular columns. .. code-block:: python @@ -298,10 +301,10 @@ support filtering partitions and selecting particular columns. pyarrow.Table value: string -Converting to a PyArrow Dataset allows you to filter on columns other than +Converting to a PyArrow Dataset allows you to filter on columns other than partition columns and load the result as a stream of batches rather than a single -table. Convert to a dataset using :meth:`DeltaTable.to_pyarrow_dataset`. Filters -applied to datasets will use the partition values and file statistics from the +table. Convert to a dataset using :meth:`DeltaTable.to_pyarrow_dataset`. Filters +applied to datasets will use the partition values and file statistics from the Delta transaction log and push down any other filters to the scanning operation. .. code-block:: python @@ -363,7 +366,7 @@ you can pass them to ``dask.dataframe.read_parquet``: >>> df Dask DataFrame Structure: value year month day - npartitions=6 + npartitions=6 object category[known] category[known] category[known] ... ... ... ... ... ... ... ... ... @@ -388,7 +391,7 @@ Vacuuming tables ~~~~~~~~~~~~~~~~ Vacuuming a table will delete any files that have been marked for deletion. This -may make some past versions of a table invalid, so this can break time travel. +may make some past versions of a table invalid, so this can break time travel. However, it will save storage space. Vacuum will retain files in a certain window, by default one week, so time travel will still work in shorter ranges. @@ -403,8 +406,8 @@ only list the files to be deleted. Pass ``dry_run=False`` to actually delete fil >>> dt = DeltaTable("../rust/tests/data/simple_table") >>> dt.vacuum() - ['../rust/tests/data/simple_table/part-00006-46f2ff20-eb5d-4dda-8498-7bfb2940713b-c000.snappy.parquet', - '../rust/tests/data/simple_table/part-00190-8ac0ae67-fb1d-461d-a3d3-8dc112766ff5-c000.snappy.parquet', + ['../rust/tests/data/simple_table/part-00006-46f2ff20-eb5d-4dda-8498-7bfb2940713b-c000.snappy.parquet', + '../rust/tests/data/simple_table/part-00190-8ac0ae67-fb1d-461d-a3d3-8dc112766ff5-c000.snappy.parquet', '../rust/tests/data/simple_table/part-00164-bf40481c-4afd-4c02-befa-90f056c2d77a-c000.snappy.parquet', ...] >>> dt.vacuum(dry_run=False) # Don't run this unless you are sure! @@ -458,16 +461,16 @@ DataFrame, a PyArrow Table, or an iterator of PyArrow Record Batches. .. code-block:: python - >>> from deltalake.writer import write_deltalake + >>> from deltalake import write_deltalake >>> df = pd.DataFrame({'x': [1, 2, 3]}) >>> write_deltalake('path/to/table', df) .. note:: - :py:func:`write_deltalake` accepts a Pandas DataFrame, but will convert it to - a Arrow table before writing. See caveats in :doc:`pyarrow:python/pandas`. + :py:func:`write_deltalake` accepts a Pandas DataFrame, but will convert it to + a Arrow table before writing. See caveats in :doc:`pyarrow:python/pandas`. -By default, writes create a new table and error if it already exists. This is -controlled by the ``mode`` parameter, which mirrors the behavior of Spark's +By default, writes create a new table and error if it already exists. This is +controlled by the ``mode`` parameter, which mirrors the behavior of Spark's :py:meth:`pyspark.sql.DataFrameWriter.saveAsTable` DataFrame method. To overwrite pass in ``mode='overwrite'`` and to append pass in ``mode='append'``: @@ -477,10 +480,57 @@ to append pass in ``mode='append'``: >>> write_deltalake('path/to/table', df, mode='append') :py:meth:`write_deltalake` will raise :py:exc:`ValueError` if the schema of -the data passed to it differs from the existing table's schema. If you wish to +the data passed to it differs from the existing table's schema. If you wish to alter the schema as part of an overwrite pass in ``overwrite_schema=True``. +Updating Delta Tables +--------------------- + +.. py:currentmodule:: deltalake.table + +Row values in an existing delta table can be updated with the :meth:`DeltaTable.update` command. A update +dictionary has to be passed, where they key is the column you wish to update, and the value is a +Expression in string format. + +Update all the rows for the column "processed" to the value True. + +.. code-block:: python + + >>> from deltalake import write_deltalake, DeltaTable + >>> df = pd.DataFrame({'x': [1, 2, 3], 'deleted': [False, False, False]}) + >>> write_deltalake('path/to/table', df) + >>> dt = DeltaTable('path/to/table') + >>> dt.update({"processed": "True"}) + >>> dt.to_pandas() + >>> x processed + 0 1 True + 1 2 True + 2 3 True +.. note:: + :meth:`DeltaTable.update` predicates and updates are all in string format. The predicates and expressions, + are parsed into Apache Datafusion expressions. + +Apply a soft deletion based on a predicate, so update all the rows for the column "deleted" to the value +True where x = 3 + +.. code-block:: python + + >>> from deltalake import write_deltalake, DeltaTable + >>> df = pd.DataFrame({'x': [1, 2, 3], 'deleted': [False, False, False]}) + >>> write_deltalake('path/to/table', df) + >>> dt = DeltaTable('path/to/table') + >>> dt.update( + ... updates={"deleted": "True"}, + ... predicate= 'x = 3', + ... ) + >>> dt.to_pandas() + >>> x deleted + 0 1 False + 1 2 False + 2 3 True + + Overwriting a partition ~~~~~~~~~~~~~~~~~~~~~~~ @@ -492,7 +542,7 @@ the method will raise an error. .. code-block:: python - >>> from deltalake.writer import write_deltalake + >>> from deltalake import write_deltalake >>> df = pd.DataFrame({'x': [1, 2, 3], 'y': ['a', 'a', 'b']}) >>> write_deltalake('path/to/table', df, partition_by=['y']) @@ -510,6 +560,39 @@ This method could also be used to insert a new partition if one doesn't already exist, making this operation idempotent. +Removing data +~~~~~~~~~~~~~ + +.. py:currentmodule:: deltalake.table + +You can remove rows from a table with :meth:`DeltaTable.delete`. A SQL where clause can +be provided to only remove some rows. If the clause matches some partition values, then +the files under those partition values will be removed. If the clause matches rows +inside some files, then those files will rewritten without the matched rows. Omitting +the clause will remove all files from the table. + +.. code-block:: python + + >>> from deltalake import DeltaTable, write_deltalake + >>> df = pd.DataFrame({'a': [1, 2, 3], 'to_delete': [False, False, True]}) + >>> write_deltalake('path/to/table', df) + + >>> table = DeltaTable('path/to/table') + >>> table.delete(predicate="to_delete = true") + {'num_added_files': 1, 'num_removed_files': 1, 'num_deleted_rows': 1, 'num_copied_rows': 2, 'execution_time_ms': 11081, 'scan_time_ms': 3721, 'rewrite_time_ms': 7} + + >>> table.to_pandas() + a to_delete + 0 1 False + 1 2 False + +.. note:: + + :meth:`DeltaTable.delete` does not delete files from storage but only updates the + table state to one where the deleted rows are no longer present. See + `Vacuuming tables`_ for more information. + + Restoring tables ~~~~~~~~~~~~~~~~ diff --git a/python/api_reference.html b/python/api_reference.html index ba3dad4a71..d5bee0b972 100644 --- a/python/api_reference.html +++ b/python/api_reference.html @@ -33,7 +33,7 @@ delta-rs
- 0.11.0 + 0.12.0
@@ -99,6 +99,27 @@

API Reference +
+delete(predicate=None)
+

Delete records from a Delta Table that statisfy a predicate.

+

When a predicate is not provided then all records are deleted from the Delta +Table. Otherwise a scan of the Delta table is performed to mark any files +that contain records that satisfy the predicate. Once files are determined +they are rewritten without the records.

+
+
Parameters
+

predicate (Optional[str]) – a SQL where clause. If not passed, will delete all rows.

+
+
Returns
+

the metrics from delete.

+
+
Return type
+

Dict[str, Any]

+
+
+
+
file_uris(partition_filters=None)
@@ -121,9 +142,9 @@

API Reference
Parameters
-

partition_filters (Optional[List[Tuple[str, str, Any]]]) – the partition filters that will be used for +

partition_filters (Optional[List[Tuple[str, str, Any]]]) – the partition filters that will be used for getting the matched files

Returns
-

list of the .parquet files referenced for the current version +

list of the .parquet files referenced for the current version of the DeltaTable

Return type
@@ -155,9 +176,9 @@

API Reference -
-files_by_partitions(partition_filters)
-

Get the files that match a given list of partitions filters.

-
-

Deprecated since version 0.7.0: Use file_uris() instead.

-
-

Partitions which do not match the filter predicate will be removed from scanned data. -Predicates are expressed in disjunctive normal form (DNF), like [(“x”, “=”, “a”), …]. -DNF allows arbitrary boolean logical combinations of single partition predicates. -The innermost tuples each describe a single partition predicate. -The list of inner predicates is interpreted as a conjunction (AND), forming a more selective and multiple partition predicates. -Each tuple has format: (key, op, value) and compares the key with the value. -The supported op are: =, !=, in, and not in. -If the op is in or not in, the value must be a collection such as a list, a set or a tuple. -The supported type for value is str. Use empty string ‘’ for Null partition value.

-

Examples: -(“x”, “=”, “a”) -(“x”, “!=”, “a”) -(“y”, “in”, [“a”, “b”, “c”]) -(“z”, “not in”, [“a”,”b”])

-
-
Parameters
-

partition_filters (List[Tuple[str, str, Any]]) – the partition filters that will be used for getting the matched files

-
-
Returns
-

list of the .parquet files after applying the partition filters referenced for the current version of the DeltaTable.

-
-
Return type
-

List[str]

-
-
-

-
classmethod from_data_catalog(data_catalog, database_name, table_name, data_catalog_id=None, version=None, log_buffer_size=None)
@@ -317,6 +304,38 @@

API Reference +
+merge(source, predicate, source_alias=None, target_alias=None, error_on_type_mismatch=True)
+

Pass the source data which you want to merge on the target delta table, providing a +predicate in SQL query like format. You can also specify on what to do when the underlying data types do not +match the underlying table.

+
+
Args:

source (pyarrow.Table | pyarrow.RecordBatch | pyarrow.RecordBatchReader ): source data +predicate (str): SQL like predicate on how to merge +source_alias (str): Alias for the source table +target_alias (str): Alias for the target table +error_on_type_mismatch (bool): specify if merge will return error if data types are mismatching :default = True

+
+
Returns:

TableMerger: TableMerger Object

+
+
+
+
Parameters
+
+
+
Return type
+

deltalake.table.TableMerger

+
+
+

+
metadata()
@@ -345,21 +364,6 @@

API Reference -
-pyarrow_schema()
-

Get the current schema of the DeltaTable with the Parquet PyArrow format.

-

DEPRECATED: use DeltaTable.schema().to_pyarrow() instead.

-
-
Returns
-

the current Schema with the Parquet PyArrow format

-
-
Return type
-

pyarrow.lib.Schema

-
-
-

-
restore(target, *, ignore_missing_files=False, protocol_downgrade_allowed=False)
@@ -390,7 +394,7 @@

API Reference

the current Schema registered in the transaction log

Return type
-

deltalake.schema.Schema

+

deltalake._internal.Schema

@@ -455,6 +459,56 @@

API Reference +
+update(updates, predicate=None, writer_properties=None, error_on_type_mismatch=True)
+

UPDATE records in the Delta Table that matches an optional predicate.

+
+
Parameters
+
    +

  • updates (Dict[str, str]) – a mapping of column name to update SQL expression.

    • +

  • predicate (Optional[str]) – a logical expression, defaults to None

    • +

  • writer_properties (Optional[Dict[str, int]]) –

    • +

  • error_on_type_mismatch (bool) –

    • +
+
+
Writer_properties
+

Pass writer properties to the Rust parquet writer, see options https://arrow.apache.org/rust/parquet/file/properties/struct.WriterProperties.html, +only the fields: data_page_size_limit, dictionary_page_size_limit, data_page_row_count_limit, write_batch_size, max_row_group_size are supported.

+
+
Error_on_type_mismatch
+

specify if merge will return error if data types are mismatching :default = True

+
+
Returns
+

the metrics from delete

+
+
Return type
+

Dict[str, Any]

+
+
+

Examples:

+

Update some row values with SQL predicate. This is equivalent to +UPDATE table SET deleted = true WHERE id = '5'

+
>>> from deltalake import DeltaTable
+>>> dt = DeltaTable("tmp")
+>>> dt.update(predicate="id = '5'",
+...           updates = {
+...             "deleted": True,
+...             }
+...         )
+
+
+

Update all row values. This is equivalent to +UPDATE table SET id = concat(id, '_old'). +>>> from deltalake import DeltaTable +>>> dt = DeltaTable(“tmp”) +>>> dt.update(updates = { +… “deleted”: True, +… “id”: “concat(id, ‘_old’)” +… } +… )

+
+
update_incremental()
@@ -510,7 +564,7 @@

API Reference
Parameters
-

table (RawDeltaTable) –

+

table (deltalake._internal.RawDeltaTable) –

@@ -576,6 +630,303 @@

API Reference +
+class deltalake.table.TableMerger(table, source, predicate, source_alias=None, target_alias=None, safe_cast=True)
+

API for various table MERGE commands.

+
+
Parameters
+
+
+
+
+
+execute()
+

Executes MERGE with the previously provided settings in Rust with Apache Datafusion query engine.

+
+
Returns:

Dict[str, any]: metrics

+
+
+
+
Return type
+

Dict[str, Any]

+
+
+
+ +
+
+when_matched_delete(predicate=None)
+

Delete a matched row from the table only if the given predicate (if specified) is +true for the matched row. If not specified it deletes all matches.

+
+
Args:

predicate (str | None, optional): SQL like predicate on when to delete. Defaults to None.

+
+
Returns:

TableMerger: TableMerger Object

+
+
+

Examples:

+

Delete on a predicate

+
>>> from deltalake import DeltaTable
+>>> import pyarrow as pa
+>>> data = pa.table({"x": [1, 2, 3], "y": [4, 5, 6]})
+>>> dt = DeltaTable("tmp")
+>>> dt.merge(source=data, predicate='target.x = source.x', source_alias='source', target_alias='target')         ...     .when_matched_delete(predicate = "source.deleted = true")
+...     .execute()
+
+
+

Delete all records that were matched

+
>>> from deltalake import DeltaTable
+>>> import pyarrow as pa
+>>> data = pa.table({"x": [1, 2, 3], "y": [4, 5, 6]})
+>>> dt = DeltaTable("tmp")
+>>> dt.merge(source=data, predicate='target.x = source.x', source_alias='source', target_alias='target')          ...     .when_matched_delete()
+...     .execute()
+
+
+
+
Parameters
+

predicate (Optional[str]) –

+
+
Return type
+

deltalake.table.TableMerger

+
+
+
+ +
+
+when_matched_update(updates, predicate=None)
+

Update a matched table row based on the rules defined by updates. +If a predicate is specified, then it must evaluate to true for the row to be updated.

+
+
Args:

updates (dict): a mapping of column name to update SQL expression. +predicate (str | None, optional): SQL like predicate on when to update. Defaults to None.

+
+
Returns:

TableMerger: TableMerger Object

+
+
+

Examples:

+
>>> from deltalake import DeltaTable
+>>> import pyarrow as pa
+>>> data = pa.table({"x": [1, 2, 3], "y": [4, 5, 6]})
+>>> dt = DeltaTable("tmp")
+>>> dt.merge(source=data, predicate='target.x = source.x', source_alias='source', target_alias='target')         ...     .when_matched_update(
+...         updates = {
+...             "x": "source.x",
+...             "y": "source.y"
+...             }
+...         ).execute()
+
+
+
+
Parameters
+
    +

  • updates (Dict[str, str]) –

    • +

  • predicate (Optional[str]) –

    • +
+
+
Return type
+

deltalake.table.TableMerger

+
+
+
+ +
+
+when_matched_update_all(predicate=None)
+

Updating all source fields to target fields, source and target are required to have the same field names. +If a predicate is specified, then it must evaluate to true for the row to be updated.

+
+
Args:

predicate (str | None, optional): SQL like predicate on when to update all columns. Defaults to None.

+
+
Returns:

TableMerger: TableMerger Object

+
+
+

Examples:

+
>>> from deltalake import DeltaTable
+>>> import pyarrow as pa
+>>> data = pa.table({"x": [1, 2, 3], "y": [4, 5, 6]})
+>>> dt = DeltaTable("tmp")
+>>> dt.merge(source=data, predicate='target.x = source.x', source_alias='source', target_alias='target')          ...     .when_matched_update_all().execute()
+
+
+
+
Parameters
+

predicate (Optional[str]) –

+
+
Return type
+

deltalake.table.TableMerger

+
+
+
+ +
+
+when_not_matched_by_source_delete(predicate=None)
+

Delete a target row that has no matches in the source from the table only if the given +predicate (if specified) is true for the target row.

+
+
Args:

updates (dict): a mapping of column name to update SQL expression. +predicate (str | None, optional): SQL like predicate on when to delete when not matched by source. Defaults to None.

+
+
Returns:

TableMerger: TableMerger Object

+
+
+
+
Parameters
+

predicate (Optional[str]) –

+
+
Return type
+

deltalake.table.TableMerger

+
+
+
+ +
+
+when_not_matched_by_source_update(updates, predicate=None)
+

Update a target row that has no matches in the source based on the rules defined by updates. +If a predicate is specified, then it must evaluate to true for the row to be updated.

+
+
Args:

updates (dict): a mapping of column name to update SQL expression. +predicate (str | None, optional): SQL like predicate on when to update. Defaults to None.

+
+
Returns:

TableMerger: TableMerger Object

+
+
+
>>> from deltalake import DeltaTable
+>>> import pyarrow as pa
+>>> data = pa.table({"x": [1, 2, 3], "y": [4, 5, 6]})
+>>> dt = DeltaTable("tmp")
+>>> dt.merge(source=data, predicate='target.x = source.x', source_alias='source', target_alias='target')         ...     .when_not_matched_by_source_update(
+...         predicate = "y > 3"
+...         updates = {
+...             "y": "0",
+...             }
+...         ).execute()    
+
+
+
+
Parameters
+
    +

  • updates (Dict[str, str]) –

    • +

  • predicate (Optional[str]) –

    • +
+
+
Return type
+

deltalake.table.TableMerger

+
+
+
+ +
+
+when_not_matched_insert(updates, predicate=None)
+

Insert a new row to the target table based on the rules defined by updates. If a +predicate is specified, then it must evaluate to true for the new row to be inserted.

+
+
Args:

updates (dict): a mapping of column name to insert SQL expression. +predicate (str | None, optional): SQL like predicate on when to insert. Defaults to None.

+
+
Returns:

TableMerger: TableMerger Object

+
+
+

Examples:

+
>>> from deltalake import DeltaTable
+>>> import pyarrow as pa
+>>> data = pa.table({"x": [1, 2, 3], "y": [4, 5, 6]})
+>>> dt = DeltaTable("tmp")
+>>> dt.merge(source=data, predicate='target.x = source.x', source_alias='source', target_alias='target')          ...     .when_not_matched_insert(
+...         updates = {
+...             "x": "source.x",
+...             "y": "source.y"
+...             }
+...         ).execute()
+
+
+
+
Parameters
+
    +

  • updates (Dict[str, str]) –

    • +

  • predicate (Optional[str]) –

    • +
+
+
Return type
+

deltalake.table.TableMerger

+
+
+
+ +
+
+when_not_matched_insert_all(predicate=None)
+

Insert a new row to the target table, updating all source fields to target fields. Source and target are +required to have the same field names. If a predicate is specified, then it must evaluate to true for +the new row to be inserted.

+
+
Args:

predicate (str | None, optional): SQL like predicate on when to insert. Defaults to None.

+
+
Returns:

TableMerger: TableMerger Object

+
+
+

Examples:

+
>>> from deltalake import DeltaTable
+>>> import pyarrow as pa
+>>> data = pa.table({"x": [1, 2, 3], "y": [4, 5, 6]})
+>>> dt = DeltaTable("tmp")
+>>> dt.merge(source=data, predicate='target.x = source.x', source_alias='source', target_alias='target')          ...     .when_not_matched_insert_all().execute()
+
+
+
+
Parameters
+

predicate (Optional[str]) –

+
+
Return type
+

deltalake.table.TableMerger

+
+
+
+ +
+
+with_writer_properties(data_page_size_limit=None, dictionary_page_size_limit=None, data_page_row_count_limit=None, write_batch_size=None, max_row_group_size=None)
+

Pass writer properties to the Rust parquet writer, see options https://arrow.apache.org/rust/parquet/file/properties/struct.WriterProperties.html:

+
+
Args:

data_page_size_limit (int|None, optional): Limit DataPage size to this in bytes. Defaults to None. +dictionary_page_size_limit (int|None, optional): Limit the size of each DataPage to store dicts to this amount in bytes. Defaults to None. +data_page_row_count_limit (int|None, optional): Limit the number of rows in each DataPage. Defaults to None. +write_batch_size (int|None, optional): Splits internally to smaller batch size. Defaults to None. +max_row_group_size (int|None, optional): Max number of rows in row group. Defaults to None.

+
+
Returns:

TableMerger: TableMerger Object

+
+
+
+
Parameters
+
    +

  • data_page_size_limit (Optional[int]) –

    • +

  • dictionary_page_size_limit (Optional[int]) –

    • +

  • data_page_row_count_limit (Optional[int]) –

    • +

  • write_batch_size (Optional[int]) –

    • +

  • max_row_group_size (Optional[int]) –

    • +
+
+
Return type
+

deltalake.table.TableMerger

+
+
+
+ +

+
class deltalake.table.TableOptimizer(table)
@@ -669,7 +1020,7 @@

API Reference

-deltalake.write_deltalake(table_or_uri, data, *, schema=None, partition_by=None, filesystem=None, mode='error', file_options=None, max_partitions=None, max_open_files=1024, max_rows_per_file=10485760, min_rows_per_group=65536, max_rows_per_group=131072, name=None, description=None, configuration=None, overwrite_schema=False, storage_options=None, partition_filters=None)
+deltalake.write_deltalake(table_or_uri, data, *, schema=None, partition_by=None, filesystem=None, mode='error', file_options=None, max_partitions=None, max_open_files=1024, max_rows_per_file=10485760, min_rows_per_group=65536, max_rows_per_group=131072, name=None, description=None, configuration=None, overwrite_schema=False, storage_options=None, partition_filters=None, large_dtypes=False)

Write to a Delta Lake table

If the table does not already exist, it will be created.

This function only supports writer protocol version 2 currently. When @@ -717,6 +1068,7 @@

Writing DeltaTablesReturn type diff --git a/python/genindex.html b/python/genindex.html index f52b29f208..52ab3cbf5f 100644 --- a/python/genindex.html +++ b/python/genindex.html @@ -31,7 +31,7 @@ delta-rs
- 0.11.0 + 0.12.0
@@ -130,6 +130,8 @@

D

  • DataCatalog (class in deltalake.data_catalog) +
    • +
  • delete() (deltalake.table.DeltaTable method)
  • deltalake.data_catalog @@ -167,6 +169,10 @@

    E

    +
    @@ -181,8 +187,6 @@

    F

  • file_uris() (deltalake.table.DeltaTable method)
  • files() (deltalake.table.DeltaTable method) -
    • -
  • files_by_partitions() (deltalake.table.DeltaTable method)
  • from_data_catalog() (deltalake.table.DeltaTable class method)
    • @@ -283,17 +287,19 @@

    M

    @@ -383,6 +387,8 @@

    S

    T

    @@ -467,6 +475,24 @@

    V

    W

    + diff --git a/python/index.html b/python/index.html index 02b6994802..098fabc111 100644 --- a/python/index.html +++ b/python/index.html @@ -33,7 +33,7 @@ delta-rs
    - 0.11.0 + 0.12.0
    @@ -100,6 +100,7 @@

    Python deltalake packageQuerying Delta Tables
  • Managing Delta Tables
  • Writing Delta Tables
    • +
  • Updating Delta Tables
  • API Reference
      diff --git a/python/installation.html b/python/installation.html index c5334032e3..e142761004 100644 --- a/python/installation.html +++ b/python/installation.html @@ -34,7 +34,7 @@ delta-rs
      - 0.11.0 + 0.12.0
      diff --git a/python/objects.inv b/python/objects.inv index c7f83f461c326ae46c6294bc6710e0e46d2ba6bd..d5ada9c8db3f06b5902b6a75aba450d88ab733f6 100644 GIT binary patch delta 1007 zcmVutI~X#Nyy;`5QlpZF#^NCd#`FUm$-JDEVq?R4o%9p-F$Tbkr?VcSh90 zi6bVl=j!DrJoUG(Mh~(ZOlR8CC7C>tPT3qBu+o1>y${S<^H$>a_a$b$cK7Zkp!neb z{&|K)0~hW4HAr#)?J0o3IU`PoTI&v&t*W zh+>b9RYIZU>n)x>W2{Ia{+Zm*S$- z%7IXpsaDF>5q5?1OIo9H%ngRoXqC}5``2nk^HAh89T=nEh$yFX800CL%pv&sV#8Z0 zBreW`LGeVyco~G+N%BOYphhCW=McO|B83uSCj#B6l+e05`K*7I62ogoq4;71y<*2Q zc!adL6Go0H<(SS9WQ#1zCNf7-jJt6{P6QAmae+lyyg7~%O;@ChW)T+eviVr6Brew! zLUG8z5$Xs=fO3}RkAgZON~D|-B#$g~fGO&t3foh1_6c_#yx|QAF6{K%jTO(`+4w(@`?dsJ>Ixn>3z&V4`39YEs z9fbAGZbzg@4V)d(Sg|$_mQioQ9fHr$iJBS2#*ncI>wbwSYN5OiIGX7Ovi|6@Onx)Mk;#3YJcez=B+PiCF7P{+ZCV|f zV%`p@SPg&9S8&eK4@@_a>DBHhNCr}7!7A-e=$DRJ-I@|}x?=eO867iyfKb*a;{Sxr z;`4AG8**tV8zSvs+*3dMx>}1L>FIO=`0b-^{7IN@O7Vkf!*S(rTfI-%>3a}Yt?Eza zx8<|pt#$l8*X8nWtK0kC!}jlSXtPHp6vywqNWxxz0d0Qs?xlXn0?5g7R+FY z233Li!Bz5Ie7Am|tyg>!kl*<4PiI&m!P@NO6zgq}{QBoMfWSE;c8z~S671pY?ftLW zxY922OKJ0)@UV5-z#hMV;u6g)uS_HIJvvqjiA<0{Ms03@7o$@1?Ezm7g{+G$}-hRIXlB%;e1FdRF1imVKiE4 zbj5yKt!N&KoVo>L^b>y(<#Y;zJSCAi1V4WkDEX4?>4e1K3>XwoM2wd~m^(?H$Q4vb zB={PFog`8yDfU93JC%}JcTg{#nod$;dW|R)U-Y0yY*_-2kb*m4q!?0);T%CW$dW7~ zV>Q_cvIN1SO2x3nbB>bwlg({g_fK(tJ`0HyH_El?v&Dub33v+K7(vrnI$ zG9H}nd&rxUi%GBM=ct9y{;yl_4T&KVRbtUCzn zPrfaYA~kS!L}P!&+@hF9JqdRRK0_xeBm#%$4g#h&+UY_|Dkj!Dvo9+s0cJ1+kLsW# zri7^s6bGAtb8=A#nnjlmc%JW; zn#X>`l(7rqeu>B{p}Y<_n&Bcae=jmkelo(5$z7d%^SMxo=b889QvjdxRqa>cB^7r; zdS=>ioO!|C`GmbZ2XWb`Zb$HM`KURwj$d(IWb3K^+4bf&``$OL_Nav7_@@&|*r5!v zzBMXt!_QgOx!Z|7=&@=2@KiRi8j45QqZe7Yova~^{~V5{Yl!_G!q3%T=K;_zvi+eT diff --git a/python/py-modindex.html b/python/py-modindex.html index 76dd4b8c85..2d573fccfe 100644 --- a/python/py-modindex.html +++ b/python/py-modindex.html @@ -34,7 +34,7 @@ delta-rs
      - 0.11.0 + 0.12.0
      diff --git a/python/search.html b/python/search.html index 6af1db9ec4..ab6ad805a4 100644 --- a/python/search.html +++ b/python/search.html @@ -34,7 +34,7 @@ delta-rs
      - 0.11.0 + 0.12.0
      diff --git a/python/searchindex.js b/python/searchindex.js index f2e3769809..613c1ab62c 100644 --- a/python/searchindex.js +++ b/python/searchindex.js @@ -1 +1 @@ -Search.setIndex({docnames:["api_reference","index","installation","usage"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":5,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.intersphinx":1,sphinx:56},filenames:["api_reference.rst","index.rst","installation.rst","usage.rst"],objects:{"deltalake.data_catalog":[[0,1,1,"","DataCatalog"]],"deltalake.data_catalog.DataCatalog":[[0,2,1,"","AWS"],[0,2,1,"","UNITY"]],"deltalake.fs":[[0,1,1,"","DeltaStorageHandler"]],"deltalake.fs.DeltaStorageHandler":[[0,3,1,"","get_file_info_selector"],[0,3,1,"","open_input_file"],[0,3,1,"","open_input_stream"],[0,3,1,"","open_output_stream"]],"deltalake.schema":[[0,1,1,"","ArrayType"],[0,1,1,"","Field"],[0,1,1,"","MapType"],[0,1,1,"","PrimitiveType"],[0,1,1,"","Schema"],[0,1,1,"","StructType"]],"deltalake.schema.ArrayType":[[0,2,1,"","contains_null"],[0,2,1,"","element_type"],[0,3,1,"","from_json"],[0,3,1,"","from_pyarrow"],[0,3,1,"","to_json"],[0,3,1,"","to_pyarrow"],[0,2,1,"","type"]],"deltalake.schema.Field":[[0,3,1,"","from_json"],[0,3,1,"","from_pyarrow"],[0,2,1,"","metadata"],[0,2,1,"","name"],[0,2,1,"","nullable"],[0,3,1,"","to_json"],[0,3,1,"","to_pyarrow"],[0,2,1,"","type"]],"deltalake.schema.MapType":[[0,3,1,"","from_json"],[0,3,1,"","from_pyarrow"],[0,2,1,"","key_type"],[0,3,1,"","to_json"],[0,3,1,"","to_pyarrow"],[0,2,1,"","type"],[0,2,1,"","value_contains_null"],[0,2,1,"","value_type"]],"deltalake.schema.PrimitiveType":[[0,3,1,"","from_json"],[0,3,1,"","from_pyarrow"],[0,3,1,"","to_json"],[0,3,1,"","to_pyarrow"],[0,2,1,"","type"]],"deltalake.schema.Schema":[[0,3,1,"","from_json"],[0,3,1,"","from_pyarrow"],[0,2,1,"","invariants"],[0,3,1,"","json"],[0,3,1,"","to_json"],[0,3,1,"","to_pyarrow"]],"deltalake.schema.StructType":[[0,2,1,"","fields"],[0,3,1,"","from_json"],[0,3,1,"","from_pyarrow"],[0,3,1,"","to_json"],[0,3,1,"","to_pyarrow"],[0,2,1,"","type"]],"deltalake.table":[[0,1,1,"","DeltaTable"],[0,1,1,"","Metadata"],[0,1,1,"","ProtocolVersions"],[0,1,1,"","TableOptimizer"]],"deltalake.table.DeltaTable":[[0,3,1,"","file_uris"],[0,3,1,"","files"],[0,3,1,"","files_by_partitions"],[0,3,1,"","from_data_catalog"],[0,3,1,"","get_add_actions"],[0,3,1,"","history"],[0,3,1,"","load_version"],[0,3,1,"","load_with_datetime"],[0,3,1,"","metadata"],[0,3,1,"","protocol"],[0,3,1,"","pyarrow_schema"],[0,3,1,"","restore"],[0,3,1,"","schema"],[0,3,1,"","to_pandas"],[0,3,1,"","to_pyarrow_dataset"],[0,3,1,"","to_pyarrow_table"],[0,3,1,"","update_incremental"],[0,3,1,"","vacuum"],[0,3,1,"","version"]],"deltalake.table.Metadata":[[0,4,1,"","configuration"],[0,4,1,"","created_time"],[0,4,1,"","description"],[0,4,1,"","id"],[0,4,1,"","name"],[0,4,1,"","partition_columns"]],"deltalake.table.ProtocolVersions":[[0,2,1,"","min_reader_version"],[0,2,1,"","min_writer_version"]],"deltalake.table.TableOptimizer":[[0,3,1,"","compact"],[0,3,1,"","z_order"]],deltalake:[[0,0,0,"-","data_catalog"],[0,0,0,"-","fs"],[0,0,0,"-","table"],[0,5,1,"","write_deltalake"]]},objnames:{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","attribute","Python attribute"],"3":["py","method","Python method"],"4":["py","property","Python property"],"5":["py","function","Python function"]},objtypes:{"0":"py:module","1":"py:class","2":"py:attribute","3":"py:method","4":"py:property","5":"py:function"},terms:{"0":[0,3],"00":[0,3],"00000":3,"00001":3,"00006":3,"0001":0,"00164":3,"00190":3,"01":0,"03":3,"04":3,"04ec9591":3,"05":3,"06":3,"07":3,"071":0,"08":0,"09":0,"09z":0,"0b73":3,"1":[0,3],"10":0,"100":3,"1000000":3,"1024":0,"10485760":0,"11":3,"12":[0,3],"131072":0,"140409099470144":3,"15":3,"1587968585495":3,"1587968586154":3,"1587968596254":3,"1587968604143":3,"1587968614187":3,"1587968626537":3,"16":3,"1811":3,"1970":0,"1994":3,"19_nyt":3,"19t16":0,"2":[0,3],"20":[0,3],"2018":0,"2020":3,"2021":3,"20gb":0,"21474836480":0,"22":3,"23":3,"2473439":3,"256mb":0,"262":3,"26t18":0,"283":3,"3":[0,3],"30":[0,3],"30b8":3,"31":0,"325440":3,"3339":0,"362":3,"39":0,"4":[0,3],"40":0,"429":3,"43f6":3,"440":3,"445":3,"453":0,"459e":3,"45fb":0,"461d":3,"4662":3,"46c8":3,"46f2ff20":3,"4758":3,"4828104688":3,"4965":3,"4acb":3,"4afd":3,"4c02":3,"4dda":3,"4fb8":3,"5":[0,3],"555":3,"565":0,"57":0,"5d5f5669530b":0,"5e68c265498":3,"5fba94":3,"6":[0,3],"6190485":3,"625":3,"62800498333851":3,"65536":0,"697l":3,"6ee3c0d22af9":3,"7":[0,3],"73e6":3,"773810":3,"7bfb2940713b":3,"7c2deba3":3,"7eb62007a15c":3,"8":3,"815e":3,"8498":3,"85f0":3,"8601":0,"8620":3,"886d":0,"895702":3,"8ac0ae67":3,"8d18":3,"8dc112766ff5":3,"9":3,"90f056c2d77a":3,"911a94a2":3,"91820cbf":0,"93ba":3,"9794":3,"9999":0,"boolean":0,"break":3,"byte":0,"case":[0,3],"class":0,"default":[0,3],"do":0,"final":3,"float":0,"function":[0,3],"import":[0,3],"int":0,"long":[0,3],"new":[0,3],"null":0,"return":[0,3],"short":0,"static":[0,2],"throw":0,"true":[0,3],"try":3,"while":[0,3],A:[0,3],AND:[0,3],AS:3,And:3,By:3,For:[1,3],If:[0,1,3],In:3,It:[1,3],NOT:0,Or:0,That:0,The:[0,3],Then:3,There:0,These:3,To:3,Will:0,_dataset:0,_dataset_parquet:0,_f:0,a3d3:3,abf:3,about:3,absolut:0,accept:[0,3],access:0,accident:3,accumul:0,achiev:3,action:0,activ:[0,1,3],actual:3,ad28:3,adb:3,add:0,addfil:3,adjust:3,adl:3,adopt:0,advantag:3,after:0,against:2,alia:0,all:[0,3],allow:[0,3],alreadi:[0,3],also:[0,3],alter:3,altern:3,alwai:3,an:[0,3],ani:[0,3],anyth:0,apach:[0,1],api:[1,3],append:[0,3],appendonli:3,appli:[0,3],ar:[0,1,2,3],arbitrari:0,argument:0,arrai:0,arraytyp:0,arrow:[0,1,3],arrow_scan:3,as_large_typ:0,assum:0,attach:0,attempt:0,authent:3,automat:3,avail:[0,3],avg:3,aw:[0,3],aws_access_key_id:3,aws_secret_access_kei:3,az:3,azur:3,azuredatabrick:3,b:[0,3],ba5711d6cb:3,ba6e:3,backend:0,base:1,basic:3,batch:[0,3],batch_it:3,batch_siz:3,bc07:3,been:[0,3],befa:3,befor:[0,3],behavior:3,being:3,belong:3,bf40481c:3,bigint:3,bin:[0,3],binari:[0,2],blob:0,bool:0,both:3,bucket:3,buffer:0,bug:1,build:0,bulk:3,c000:3,c373a5bd:3,c9b90f86:3,c:0,call:[0,3],can:[0,3],capabl:1,carri:0,cast:3,catalog:[0,3],catalog_nam:3,categori:3,caveat:3,cb6b150b:3,certain:3,chang:3,checkpoint:0,chronolog:0,classmethod:0,close:0,col:0,collect:0,column:[0,3],com:0,combin:0,command:0,commit:[0,3],common:3,commun:2,compact:[0,3],compar:[0,3],compat:[0,1,3],complet:1,compliant:3,comput:3,concaten:0,concret:0,concurr:[0,3],condit:3,configur:[0,3],conjunct:0,consid:[0,3],constructor:3,contain:[0,3],contains_nul:0,containsnul:0,control:3,convers:0,convert:[0,3],core:0,could:3,counti:3,covid:3,cpu:0,creat:[0,3],created_tim:[0,3],creation:3,critic:2,current:0,curv:0,custom_metadata:0,d46c948aa415:3,dai:3,dask:3,data:[0,3],data_catalog:[0,3],data_catalog_id:[0,3],data_chang:[0,3],data_typ:0,databas:[0,3],database_nam:[0,3],databrick:[0,3],databricks_access_token:3,databricks_workspace_url:3,datacatalog:[1,3],datafram:[0,3],dataframewrit:3,dataset:[0,3],datatyp:0,date:[0,3],datetim:[0,3],datetime_str:0,db_schema:3,db_tabl:3,dd:3,decim:0,decreas:0,defin:[0,3],delet:[0,3],deletedfileretentiondur:0,delta:1,deltalak:[0,2,3],deltaprotocolerror:0,deltastoragehandl:1,deltat:[1,3],depend:[1,3],deprec:0,deriv:3,describ:[0,3],descript:[0,3],desir:0,determin:[0,3],develop:1,df2:3,df:3,dict:0,dictionari:0,differ:[0,3],directori:0,disabl:0,disjunct:0,disk:0,dnf:0,doc:[0,3],document:1,doe:[0,3],doesn:[0,3],don:3,done:3,doubl:0,down:3,dry:3,dry_run:[0,3],ds:3,dt:[0,3],duckdb:[1,3],duplic:0,e:[0,3],each:0,eb5d:3,edg:0,either:[0,3],element:0,element_typ:0,elementtyp:0,els:3,empti:0,encod:3,encount:1,end:0,enforce_retention_dur:0,engin:3,environ:3,epoch:0,equival:0,error:[0,3],errorifexist:3,even:3,ex_data:3,examin:1,exampl:[0,3],except:3,execut:0,exist:[0,3],expect:0,experiment:1,explicitli:0,express:[0,3],f698:0,factori:3,fail:[0,3],fals:[0,3],faster:0,fb1d:3,featur:1,ff32ddab96d2:3,ff6bfaf892a:3,field:[0,3],field_json:0,file:[0,2,3],file_opt:0,file_uri:[0,3],fileinfo:0,files_by_partit:0,filesad:3,fileselector:0,filesremov:3,filesystem:[0,3],filesystemhandl:0,filter:[0,3],first:[0,3],fit:3,flatten:[0,3],form:[0,3],format:[0,3],fragment:0,frame:[0,3],from:[0,1,3],from_data_catalog:[0,3],from_json:[0,3],from_pyarrow:0,from_uri:3,fs:[0,3],fsspec:0,g:[0,3],gc:3,gener:0,get:[0,3],get_add_act:[0,3],get_file_info_selector:0,github:[0,1,2],given:[0,3],glue:[0,3],good:3,greater:0,group:0,gs:3,ha:[0,3],handl:0,happen:0,have:[0,3],help:0,higher:0,histori:0,hood:1,hour:0,how:0,howev:3,html:[0,3],http:[0,3],id:[0,3],idempot:[0,3],identifi:[0,3],ignor:0,ignore_missing_fil:0,implement:[0,1],improv:[0,3],includ:[0,3],incom:0,increas:0,increment:[0,3],index:1,indic:[0,3],infer:0,info:0,inform:3,initi:0,inner:0,innermost:0,input:[0,3],insert:3,insid:0,instal:1,instanc:0,instead:[0,1],int32:0,int64:[0,3],int96:0,integ:0,integr:[1,3],interfac:[0,3],intern:3,interpret:0,interv:0,introspect:3,invalid:3,invari:0,io:3,irrelev:3,isblindappend:3,isn:0,iso:0,issu:2,iter:[0,3],its:3,java:1,json:[0,3],just:[0,3],jvm:1,kei:0,key_typ:0,keytyp:0,know:1,known:3,known_siz:0,l492:0,l533:0,lake:[1,3],larg:[0,3],larger:3,last:0,latenc:0,latest:[0,3],latter:3,least:0,left:0,let:1,level:[0,3],lib:0,librari:1,like:[0,3],limit:0,link:2,list:[0,3],listtyp:0,liter:0,load:[0,1],load_vers:[0,3],load_with_datetim:[0,3],local:0,localfilesystem:3,locat:0,log:[0,3],log_buffer_s:0,logic:0,logretentiondur:3,longer:0,look:3,low:0,made:0,mai:[0,3],main:3,maintain:3,make:[0,3],make_write_opt:0,manag:1,mani:[0,3],map:[0,3],maptyp:0,mark:3,master:0,match:[0,3],max:[0,3],max_concurrent_task:0,max_open_fil:0,max_partit:0,max_rows_per_fil:0,max_rows_per_group:0,max_spill_s:0,maximum:0,meant:3,memori:[0,3],merg:3,metadata:0,method:3,metric:0,millisecond:0,min:[0,3],min_commit_interv:0,min_reader_vers:0,min_rows_per_group:0,min_writer_vers:0,minimum:0,minut:0,mirror:3,miss:[0,3],mode:[0,3],modification_tim:[0,3],modul:[1,3],month:3,more:[0,3],multipl:[0,3],must:[0,3],my_col:0,my_int_col:0,name:[0,3],nativ:[0,1],nativefil:0,need:[0,3],nest:0,net:3,newdata:3,newer:0,none:[0,3],normal:[0,3],normalized_path:3,note:[0,2,3],noth:0,npartit:3,null_count:[0,3],nullabl:[0,3],num_record:[0,3],numbatch:3,number:[0,3],numfilesad:3,numfilesremov:3,numremovedfil:3,numrestoredfil:3,object:[0,2,3],offici:2,often:3,old:3,olddata:3,older:0,onc:3,one:[0,3],onli:[0,3],op:0,open:0,open_input_fil:0,open_input_stream:0,open_output_stream:0,openssl:2,oper:[0,3],operationparamet:3,optim:0,option:[0,3],order:[0,3],org:0,other:[0,1,3],otherwis:[0,3],our:1,output:0,overwrit:0,overwrite_schema:[0,3],p:0,pa:0,pack:[0,3],page:1,panda:[0,1,3],paramet:[0,3],parquet:[0,3],parquet_read_opt:0,parquetfilewriteopt:0,parquetreadopt:0,pars:0,part:3,particular:3,partit:0,partition_bi:[0,3],partition_column:[0,3],partition_filt:[0,3],partition_valu:0,partitionbi:3,partitioncolumn:3,partitionsoptim:3,pass:[0,3],past:3,path:[0,3],pathlib:0,pd:3,per:0,perform:[0,3],pip:1,place:0,plain:0,pleas:[0,1,2],point:[0,3],polar:[0,1],portion:3,posit:0,possibl:0,practic:3,precis:0,predic:[0,3],prefix:0,preserv:0,preserveinsertionord:3,prevent:3,preview:3,previou:3,primit:0,primitivetyp:[0,3],print:3,process:3,project:[0,3],properli:3,properti:[0,3],protocol:0,protocol_downgrade_allow:0,protocolvers:0,proven:3,provid:[0,1,3],push:3,pyarrow:[0,3],pyarrow_schema:0,pyspark:[1,3],python:0,pythonfil:0,pyx:0,queri:1,quot:0,rais:[0,3],random:0,rang:3,rate:0,rather:3,raw_f:3,rawdeltat:0,read:[0,1,3],read_parquet:3,reader:0,readvers:3,recent:0,reconstruct:3,record:3,recordbatch:0,recordbatchread:0,reduc:[0,3],redund:3,refer:[1,3],referenc:0,regist:0,regularli:3,rel:0,remot:2,remov:[0,3],removefil:3,reorder:0,replac:0,repo:1,repres:[0,3],represent:[0,3],request:2,requir:[0,1,3],respect:[0,3],restor:0,result:[0,3],retain:3,retent:0,retention_hour:0,retriev:[0,3],revers:0,rfc:0,right:0,root:[0,3],row:[0,3],rs:1,run:[0,3],rust:[1,3],s3:[0,3],s3a:3,s:[0,3],same:0,save:[0,3],saveast:3,scale:0,scan:[0,3],schema:1,schema_json:0,schema_nam:3,scheme:0,search:1,second:0,see:[0,1,3],select:[0,3],selector:0,sequenti:0,serializ:0,servic:3,set:0,sever:3,shorter:3,should:0,simple_databas:3,simple_t:3,sinc:0,singl:[0,3],size:[0,3],size_byt:[0,3],skip:[0,3],slower:3,small:[0,3],smaller:0,snappi:3,so:[0,1,3],some:[0,1,3],someth:3,sometim:3,sourc:0,space:3,spark:[1,3],specif:3,specifi:[0,3],spill:0,split:0,sql:[0,3],state:3,statist:[0,3],still:3,storag:0,storage_opt:[0,3],store:[2,3],str:0,stream:[0,3],string:[0,3],struct:[0,3],structtyp:0,structur:3,subfield:0,submodul:0,subset:0,subtreefilesystem:[0,3],suffici:0,support:[0,3],sure:3,syntax:0,system:[0,3],t:[0,3],tabl:0,table_nam:[0,3],table_or_uri:0,table_uri:[0,3],tableoptim:[0,3],tag:0,target:0,target_s:0,targetfiles:0,task:[0,3],test:[0,3],than:[0,3],the_aws_access_key_id:3,the_aws_secret_access_kei:3,thei:[0,3],them:3,thi:[0,1,3],those:3,threshold:0,thrown:3,time:0,time_delta:0,timedelta:0,timestamp:[0,3],tmp:0,to_batch:3,to_json:0,to_panda:[0,3],to_pyarrow:[0,3],to_pyarrow_dataset:[0,3],to_pyarrow_t:[0,3],to_pydict:3,to_tabl:3,togeth:3,too:0,total:0,totalconsideredfil:3,totalfil:3,totalfilesskip:3,totals:3,transact:[0,3],travel:0,tree:3,truncat:0,tupl:0,twice:0,type:[0,3],type_json:0,typeerror:0,under:1,union:0,uniqu:[0,3],uniti:[0,3],unix:0,unless:[0,3],unus:0,up:0,updat:[0,3],update_increment:0,upgrad:[0,2],uri:0,url:3,us:[0,1,3],usag:[0,1],user:0,usual:3,util:3,vacuum:0,valid:0,valu:[0,3],value_contains_nul:0,value_typ:0,valuecontainsnul:0,valueerror:3,valuetyp:0,varchar:3,variabl:[0,3],variant:0,variou:0,ve:3,version:[0,3],via:3,view:3,wa:3,wai:3,want:0,we:3,week:[0,3],well:3,were:3,what:3,wheel:2,when:[0,3],whether:0,which:[0,3],whole:3,whom:3,window:3,wish:3,within:[0,3],without:[0,1],without_fil:0,work:3,world:3,would:3,wrap:3,write:1,write_deltalak:[0,3],writer:[0,3],written:3,wrote:3,x:[0,3],y:[0,3],year:3,yet:1,you:[0,1,3],your:[0,3],z:0,z_order:[0,3]},titles:["API Reference","Python deltalake package","Installation","Usage"],titleterms:{action:3,add:3,api:0,backend:3,current:3,custom:3,datacatalog:0,delta:[0,3],deltalak:1,deltastoragehandl:0,deltat:0,examin:3,histori:3,indic:1,instal:2,lake:0,load:3,manag:3,metadata:3,optim:3,overwrit:3,packag:1,partit:3,pip:2,python:1,queri:3,refer:0,restor:3,schema:[0,3],storag:3,tabl:[1,3],time:3,travel:3,us:2,usag:3,vacuum:3,write:[0,3]}}) \ No newline at end of file +Search.setIndex({docnames:["api_reference","index","installation","usage"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":5,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.intersphinx":1,sphinx:56},filenames:["api_reference.rst","index.rst","installation.rst","usage.rst"],objects:{"deltalake.data_catalog":[[0,1,1,"","DataCatalog"]],"deltalake.data_catalog.DataCatalog":[[0,2,1,"","AWS"],[0,2,1,"","UNITY"]],"deltalake.fs":[[0,1,1,"","DeltaStorageHandler"]],"deltalake.fs.DeltaStorageHandler":[[0,3,1,"","get_file_info_selector"],[0,3,1,"","open_input_file"],[0,3,1,"","open_input_stream"],[0,3,1,"","open_output_stream"]],"deltalake.schema":[[0,1,1,"","ArrayType"],[0,1,1,"","Field"],[0,1,1,"","MapType"],[0,1,1,"","PrimitiveType"],[0,1,1,"","Schema"],[0,1,1,"","StructType"]],"deltalake.schema.ArrayType":[[0,2,1,"","contains_null"],[0,2,1,"","element_type"],[0,3,1,"","from_json"],[0,3,1,"","from_pyarrow"],[0,3,1,"","to_json"],[0,3,1,"","to_pyarrow"],[0,2,1,"","type"]],"deltalake.schema.Field":[[0,3,1,"","from_json"],[0,3,1,"","from_pyarrow"],[0,2,1,"","metadata"],[0,2,1,"","name"],[0,2,1,"","nullable"],[0,3,1,"","to_json"],[0,3,1,"","to_pyarrow"],[0,2,1,"","type"]],"deltalake.schema.MapType":[[0,3,1,"","from_json"],[0,3,1,"","from_pyarrow"],[0,2,1,"","key_type"],[0,3,1,"","to_json"],[0,3,1,"","to_pyarrow"],[0,2,1,"","type"],[0,2,1,"","value_contains_null"],[0,2,1,"","value_type"]],"deltalake.schema.PrimitiveType":[[0,3,1,"","from_json"],[0,3,1,"","from_pyarrow"],[0,3,1,"","to_json"],[0,3,1,"","to_pyarrow"],[0,2,1,"","type"]],"deltalake.schema.Schema":[[0,3,1,"","from_json"],[0,3,1,"","from_pyarrow"],[0,2,1,"","invariants"],[0,3,1,"","json"],[0,3,1,"","to_json"],[0,3,1,"","to_pyarrow"]],"deltalake.schema.StructType":[[0,2,1,"","fields"],[0,3,1,"","from_json"],[0,3,1,"","from_pyarrow"],[0,3,1,"","to_json"],[0,3,1,"","to_pyarrow"],[0,2,1,"","type"]],"deltalake.table":[[0,1,1,"","DeltaTable"],[0,1,1,"","Metadata"],[0,1,1,"","ProtocolVersions"],[0,1,1,"","TableMerger"],[0,1,1,"","TableOptimizer"]],"deltalake.table.DeltaTable":[[0,3,1,"","delete"],[0,3,1,"","file_uris"],[0,3,1,"","files"],[0,3,1,"","from_data_catalog"],[0,3,1,"","get_add_actions"],[0,3,1,"","history"],[0,3,1,"","load_version"],[0,3,1,"","load_with_datetime"],[0,3,1,"","merge"],[0,3,1,"","metadata"],[0,3,1,"","protocol"],[0,3,1,"","restore"],[0,3,1,"","schema"],[0,3,1,"","to_pandas"],[0,3,1,"","to_pyarrow_dataset"],[0,3,1,"","to_pyarrow_table"],[0,3,1,"","update"],[0,3,1,"","update_incremental"],[0,3,1,"","vacuum"],[0,3,1,"","version"]],"deltalake.table.Metadata":[[0,4,1,"","configuration"],[0,4,1,"","created_time"],[0,4,1,"","description"],[0,4,1,"","id"],[0,4,1,"","name"],[0,4,1,"","partition_columns"]],"deltalake.table.ProtocolVersions":[[0,2,1,"","min_reader_version"],[0,2,1,"","min_writer_version"]],"deltalake.table.TableMerger":[[0,3,1,"","execute"],[0,3,1,"","when_matched_delete"],[0,3,1,"","when_matched_update"],[0,3,1,"","when_matched_update_all"],[0,3,1,"","when_not_matched_by_source_delete"],[0,3,1,"","when_not_matched_by_source_update"],[0,3,1,"","when_not_matched_insert"],[0,3,1,"","when_not_matched_insert_all"],[0,3,1,"","with_writer_properties"]],"deltalake.table.TableOptimizer":[[0,3,1,"","compact"],[0,3,1,"","z_order"]],deltalake:[[0,0,0,"-","data_catalog"],[0,0,0,"-","fs"],[0,0,0,"-","table"],[0,5,1,"","write_deltalake"]]},objnames:{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","attribute","Python attribute"],"3":["py","method","Python method"],"4":["py","property","Python property"],"5":["py","function","Python function"]},objtypes:{"0":"py:module","1":"py:class","2":"py:attribute","3":"py:method","4":"py:property","5":"py:function"},terms:{"0":[0,3],"00":[0,3],"00000":3,"00001":3,"00006":3,"0001":0,"00164":3,"00190":3,"01":0,"03":3,"04":3,"04ec9591":3,"05":3,"06":3,"07":3,"071":0,"08":0,"09":0,"09z":0,"0b73":3,"1":[0,3],"10":0,"100":3,"1000000":3,"1024":0,"10485760":0,"11":3,"11081":3,"12":[0,3],"131072":0,"140409099470144":3,"15":3,"1587968585495":3,"1587968586154":3,"1587968596254":3,"1587968604143":3,"1587968614187":3,"1587968626537":3,"16":3,"1811":3,"1970":0,"1994":3,"19_nyt":3,"19t16":0,"2":[0,3],"20":[0,3],"2018":0,"2020":3,"2021":3,"20gb":0,"21474836480":0,"22":3,"23":3,"2473439":3,"256mb":0,"262":3,"26t18":0,"283":3,"3":[0,3],"30":[0,3],"30b8":3,"31":0,"325440":3,"3339":0,"362":3,"3721":3,"39":0,"4":[0,3],"40":0,"429":3,"43f6":3,"440":3,"445":3,"453":0,"459e":3,"45fb":0,"461d":3,"4662":3,"46c8":3,"46f2ff20":3,"4758":3,"4828104688":3,"4965":3,"4acb":3,"4afd":3,"4c02":3,"4dda":3,"4fb8":3,"5":[0,3],"555":3,"565":0,"57":0,"5d5f5669530b":0,"5e68c265498":3,"5fba94":3,"6":[0,3],"6190485":3,"625":3,"62800498333851":3,"65536":0,"697l":3,"6ee3c0d22af9":3,"7":3,"73e6":3,"773810":3,"7bfb2940713b":3,"7c2deba3":3,"7eb62007a15c":3,"8":3,"815e":3,"8498":3,"85f0":3,"8601":0,"8620":3,"886d":0,"895702":3,"8ac0ae67":3,"8d18":3,"8dc112766ff5":3,"9":3,"90f056c2d77a":3,"911a94a2":3,"91820cbf":0,"93ba":3,"9794":3,"9999":0,"boolean":0,"break":3,"byte":0,"case":[0,3],"class":0,"default":[0,3],"do":0,"final":3,"float":0,"function":[0,3],"import":[0,3],"int":0,"long":[0,3],"new":[0,3],"null":0,"return":[0,3],"short":0,"static":[0,2],"throw":0,"true":[0,3],"try":3,"while":[0,3],A:[0,3],AND:[0,3],AS:3,And:3,By:3,For:[1,3],If:[0,1,3],In:3,It:[1,3],NOT:0,Or:0,That:0,The:[0,3],Then:3,There:0,These:3,To:3,Will:0,_dataset:0,_dataset_parquet:0,_f:0,_intern:0,_old:0,a3d3:3,abf:3,about:3,absolut:0,accept:[0,3],access:0,accident:3,accumul:0,achiev:3,action:0,activ:[0,1,3],actual:3,ad28:3,adb:3,add:0,addfil:3,adjust:3,adl:3,adopt:0,advantag:3,after:0,against:[0,2],alia:0,all:[0,3],allow:[0,3],alreadi:[0,3],also:[0,3],alter:3,altern:3,alwai:3,amount:0,an:[0,3],ani:[0,3],anyth:0,apach:[0,1,3],api:[1,3],append:[0,3],appendonli:3,appli:[0,3],ar:[0,1,2,3],arbitrari:0,arg:0,argument:0,arrai:0,arraytyp:0,arrow:[0,1,3],arrow_scan:3,as_large_typ:0,assum:0,attach:0,attempt:0,authent:3,automat:3,avail:[0,3],avg:3,aw:[0,3],aws_access_key_id:3,aws_secret_access_kei:3,az:3,azur:3,azuredatabrick:3,b:[0,3],ba5711d6cb:3,ba6e:3,backend:0,base:[0,1,3],basic:3,batch:[0,3],batch_it:3,batch_siz:3,bc07:3,been:[0,3],befa:3,befor:[0,3],behavior:3,being:3,belong:3,bf40481c:3,bigint:3,bin:[0,3],binari:[0,2],blob:0,bool:0,both:3,bucket:3,buffer:0,bug:1,build:0,bulk:3,c000:3,c373a5bd:3,c9b90f86:3,c:0,call:[0,3],can:[0,3],capabl:1,carri:0,cast:3,catalog:[0,3],catalog_nam:3,categori:3,caveat:3,cb6b150b:3,certain:3,chang:3,check:0,checkpoint:0,chronolog:0,classmethod:0,claus:[0,3],close:0,col:0,collect:0,column:[0,3],com:0,combin:0,command:[0,3],commit:[0,3],common:3,commun:2,compact:[0,3],compar:[0,3],compat:[0,1,3],complet:1,compliant:3,comput:3,concat:0,concaten:0,concret:0,concurr:[0,3],condit:3,configur:[0,3],conjunct:0,consid:[0,3],constructor:3,contain:[0,3],contains_nul:0,containsnul:0,control:3,convers:0,convert:[0,3],core:0,could:3,counti:3,covid:3,cpu:0,creat:[0,3],created_tim:[0,3],creation:3,critic:2,current:0,curv:0,custom_metadata:0,d46c948aa415:3,dai:3,dask:3,data:0,data_catalog:[0,3],data_catalog_id:[0,3],data_chang:[0,3],data_page_row_count_limit:0,data_page_size_limit:0,data_typ:0,databas:[0,3],database_nam:[0,3],databrick:[0,3],databricks_access_token:3,databricks_workspace_url:3,datacatalog:[1,3],datafram:[0,3],dataframewrit:3,datafus:[0,3],datapag:0,dataset:[0,3],datatyp:0,date:[0,3],datetim:[0,3],datetime_str:0,db_schema:3,db_tabl:3,dbat:3,dd:3,decim:0,decreas:0,defin:[0,3],delet:[0,3],deletedfileretentiondur:0,delta:1,deltalak:[0,2,3],deltaprotocolerror:0,deltastoragehandl:1,deltat:[1,3],depend:[1,3],deprec:0,deriv:3,describ:[0,3],descript:[0,3],desir:0,determin:[0,3],develop:1,df2:3,df:3,dict:0,dictionari:[0,3],dictionary_page_size_limit:0,differ:[0,3],directori:0,disabl:0,disjunct:0,disk:0,dnf:0,doc:[0,3],document:1,doe:[0,3],doesn:[0,3],don:3,done:3,doubl:0,down:3,dry:3,dry_run:[0,3],ds:3,dt:[0,3],duckdb:[1,3],duplic:0,e:[0,3],each:0,eb5d:3,edg:0,either:[0,3],element:0,element_typ:0,elementtyp:0,els:3,empti:0,encod:3,encount:1,end:0,enforce_retention_dur:0,engin:[0,3],environ:3,epoch:0,equival:0,error:[0,3],error_on_type_mismatch:0,errorifexist:3,evalu:0,even:3,ex_data:3,examin:1,exampl:[0,3],except:3,execut:0,execution_time_m:3,exist:[0,3],expect:0,experiment:1,explicitli:0,express:[0,3],f698:0,factori:3,fail:[0,3],fals:[0,3],faster:0,fb1d:3,featur:1,ff32ddab96d2:3,ff6bfaf892a:3,field:[0,3],field_json:0,file:[0,2,3],file_opt:0,file_uri:[0,3],fileinfo:0,files_by_partit:0,filesad:3,fileselector:0,filesremov:3,filesystem:[0,3],filesystemhandl:0,filter:[0,3],first:[0,3],fit:3,flatten:[0,3],form:[0,3],format:[0,3],fragment:0,frame:[0,3],from:[0,1,3],from_data_catalog:[0,3],from_json:[0,3],from_pyarrow:0,from_uri:3,fs:[0,3],fsspec:0,g:[0,3],gc:3,gener:0,get:[0,3],get_add_act:[0,3],get_file_info_selector:0,github:[0,1,2],given:[0,3],glue:[0,3],good:3,greater:0,group:0,gs:3,ha:[0,3],handl:0,happen:0,have:[0,3],help:0,higher:0,histori:0,hood:1,hour:0,how:0,howev:3,html:[0,3],http:[0,3],id:[0,3],idempot:[0,3],identifi:[0,3],ignor:0,ignore_missing_fil:0,implement:[0,1],improv:[0,3],includ:[0,3],incom:0,increas:0,increment:[0,3],index:1,indic:[0,3],infer:0,info:0,inform:3,initi:0,inner:0,innermost:0,input:[0,3],insert:[0,3],insid:[0,3],instal:1,instanc:0,instead:1,int32:0,int64:[0,3],int96:0,integ:0,integr:[1,3],interfac:[0,3],intern:[0,3],interpret:0,interv:0,introspect:3,invalid:3,invari:0,io:3,irrelev:3,isblindappend:3,isn:0,iso:0,issu:2,iter:[0,3],its:3,java:1,json:[0,3],just:[0,3],jvm:1,kei:[0,3],key_typ:0,keytyp:0,know:1,known:3,known_siz:0,l492:0,l533:0,lake:[1,3],larg:[0,3],large_dtyp:0,larger:3,last:0,latenc:0,latest:[0,3],latter:3,least:0,left:0,let:1,level:[0,3],lib:0,librari:1,like:[0,3],limit:0,link:2,list:[0,3],listtyp:0,liter:0,load:[0,1],load_vers:[0,3],load_with_datetim:[0,3],local:0,localfilesystem:3,locat:0,log:[0,3],log_buffer_s:0,logic:0,logretentiondur:3,longer:[0,3],look:3,low:0,made:0,mai:[0,3],main:3,maintain:3,make:[0,3],make_write_opt:0,manag:1,mani:[0,3],map:[0,3],maptyp:0,mark:[0,3],master:0,match:[0,3],max:[0,3],max_concurrent_task:0,max_open_fil:0,max_partit:0,max_row_group_s:0,max_rows_per_fil:0,max_rows_per_group:0,max_spill_s:0,maximum:0,meant:3,memori:[0,3],merg:[0,3],metadata:0,method:3,metric:0,millisecond:0,min:[0,3],min_commit_interv:0,min_reader_vers:0,min_rows_per_group:0,min_writer_vers:0,minimum:0,minut:0,mirror:3,mismatch:0,miss:[0,3],mode:[0,3],modification_tim:[0,3],modul:[1,3],month:3,more:[0,3],multipl:[0,3],must:[0,3],my_col:0,my_int_col:0,name:[0,3],nativ:[0,1],nativefil:0,need:[0,3],nest:0,net:3,newdata:3,newer:0,none:[0,3],normal:[0,3],normalized_path:3,note:[0,2,3],noth:0,npartit:3,null_count:[0,3],nullabl:[0,3],num_added_fil:3,num_copied_row:3,num_deleted_row:3,num_record:[0,3],num_removed_fil:3,numbatch:3,number:[0,3],numfilesad:3,numfilesremov:3,numremovedfil:3,numrestoredfil:3,object:[0,2,3],offici:2,often:3,old:3,olddata:3,older:0,omit:3,onc:[0,3],one:[0,3],onli:[0,3],op:0,open:0,open_input_fil:0,open_input_stream:0,open_output_stream:0,openssl:2,oper:[0,3],operationparamet:3,optim:0,option:[0,3],order:[0,3],org:0,os:3,other:[0,1,3],otherwis:[0,3],our:1,output:0,overwrit:0,overwrite_schema:[0,3],p:0,pa:0,pack:[0,3],page:1,panda:[0,1,3],paramet:[0,3],parquet:[0,3],parquet_read_opt:0,parquetfilewriteopt:0,parquetreadopt:0,pars:[0,3],part:3,particular:3,partit:0,partition_bi:[0,3],partition_column:[0,3],partition_filt:[0,3],partition_valu:0,partitionbi:3,partitioncolumn:3,partitionsoptim:3,pass:[0,3],past:3,path:[0,3],pathlib:0,pd:3,per:0,perform:[0,3],pip:1,place:0,plain:0,pleas:[0,1,2],point:[0,3],polar:[0,1],portion:3,posit:0,possibl:0,practic:3,precis:0,predic:[0,3],prefix:0,present:3,preserv:0,preserveinsertionord:3,prevent:3,preview:3,previou:3,previous:0,primit:0,primitivetyp:[0,3],print:3,process:3,project:[0,3],properli:3,properti:[0,3],protocol:0,protocol_downgrade_allow:0,protocolvers:0,proven:3,provid:[0,1,3],push:3,pyarrow:[0,3],pyspark:[1,3],python:0,pythonfil:0,pyx:0,queri:[0,1],quot:0,rais:[0,3],random:0,rang:3,rate:0,rather:3,raw_f:3,rawdeltat:0,read:[0,1,3],read_parquet:3,reader:0,readvers:3,recent:0,reconstruct:3,record:[0,3],recordbatch:0,recordbatchread:0,reduc:[0,3],redund:3,refer:[1,3],referenc:0,regist:0,regularli:3,rel:0,remot:2,removefil:3,reorder:0,replac:0,repo:1,repres:[0,3],represent:[0,3],request:2,requir:[0,1,3],respect:[0,3],restor:0,result:[0,3],retain:3,retent:0,retention_hour:0,retriev:[0,3],revers:0,rewrite_time_m:3,rewritten:[0,3],rfc:0,right:0,root:[0,3],row:[0,3],rs:1,rule:0,run:[0,3],rust:[0,1,3],s3:[0,3],s3a:3,s:[0,3],safe_cast:0,same:0,satisfi:0,save:[0,3],saveast:3,scale:0,scan:[0,3],scan_time_m:3,schema:1,schema_json:0,schema_nam:3,scheme:0,search:1,second:0,see:[0,1,3],select:[0,3],selector:0,sequenti:0,serializ:0,servic:3,set:0,sever:3,shorter:3,should:0,simple_databas:3,simple_t:3,sinc:0,singl:[0,3],size:[0,3],size_byt:[0,3],skip:[0,3],slower:3,small:[0,3],smaller:0,snappi:3,so:[0,1,3],soft:3,some:[0,1,3],someth:3,sometim:3,sourc:0,source_alia:0,space:3,spark:[1,3],specif:3,specifi:[0,3],spill:0,split:0,sql:[0,3],state:3,statisfi:0,statist:[0,3],still:3,storag:0,storage_opt:[0,3],store:[0,2,3],str:0,stream:[0,3],string:[0,3],struct:[0,3],structtyp:0,structur:3,subfield:0,submodul:0,subset:0,subtreefilesystem:[0,3],suffici:0,support:[0,3],sure:3,syntax:0,system:[0,3],t:[0,3],tabl:0,table_nam:[0,3],table_or_uri:0,table_uri:[0,3],tablemerg:0,tableoptim:[0,3],tag:0,target:0,target_alia:0,target_s:0,targetfiles:0,task:[0,3],test:[0,3],than:[0,3],the_aws_access_key_id:3,the_aws_secret_access_kei:3,thei:[0,3],them:3,thi:[0,1,3],those:3,threshold:0,thrown:3,time:0,time_delta:0,timedelta:0,timestamp:[0,3],tmp:0,to_batch:3,to_delet:3,to_json:0,to_panda:[0,3],to_pyarrow:[0,3],to_pyarrow_dataset:[0,3],to_pyarrow_t:[0,3],to_pydict:3,to_tabl:3,togeth:3,too:0,total:0,totalconsideredfil:3,totalfil:3,totalfilesskip:3,totals:3,transact:[0,3],travel:0,tree:3,truncat:0,tupl:0,twice:0,type:[0,3],type_json:0,typeerror:0,under:[1,3],underli:0,union:0,uniqu:[0,3],uniti:[0,3],unix:0,unless:[0,3],unus:0,up:0,updat:[0,1],update_increment:0,upgrad:[0,2],uri:0,url:3,us:[0,1,3],usag:[0,1],user:0,usual:3,util:3,vacuum:0,valid:0,valu:[0,3],value_contains_nul:0,value_typ:0,valuecontainsnul:0,valueerror:3,valuetyp:0,varchar:3,variabl:[0,3],variant:0,variou:0,ve:3,version:[0,3],via:3,view:3,wa:3,wai:3,want:0,we:3,week:[0,3],well:3,were:[0,3],what:[0,3],wheel:2,when:[0,3],when_matched_delet:0,when_matched_upd:0,when_matched_update_al:0,when_not_matched_by_source_delet:0,when_not_matched_by_source_upd:0,when_not_matched_insert:0,when_not_matched_insert_al:0,where:[0,3],whether:0,which:[0,3],whole:3,whom:3,window:3,wish:3,with_writer_properti:0,within:[0,3],without:[0,1,3],without_fil:0,work:3,world:3,would:3,wrap:3,write:1,write_batch_s:0,write_deltalak:[0,3],writer:[0,3],writer_properti:0,writerproperti:0,written:3,wrote:3,x:[0,3],y:[0,3],year:3,yet:1,you:[0,1,3],your:[0,3],z:0,z_order:[0,3]},titles:["API Reference","Python deltalake package","Installation","Usage"],titleterms:{action:3,add:3,api:0,backend:3,current:3,custom:3,data:3,datacatalog:0,delta:[0,3],deltalak:1,deltastoragehandl:0,deltat:0,examin:3,histori:3,indic:1,instal:2,lake:0,load:3,manag:3,metadata:3,optim:3,overwrit:3,packag:1,partit:3,pip:2,python:1,queri:3,refer:0,remov:3,restor:3,schema:[0,3],storag:3,tabl:[1,3],time:3,travel:3,updat:3,us:2,usag:3,vacuum:3,write:[0,3]}}) \ No newline at end of file diff --git a/python/usage.html b/python/usage.html index 1111859f3a..6420e5cf2b 100644 --- a/python/usage.html +++ b/python/usage.html @@ -34,7 +34,7 @@ delta-rs
      - 0.11.0 + 0.12.0
      @@ -65,8 +65,10 @@
    • Optimizing tables
    • -
  • Writing Delta Tables