From 835fc81a3b98d3737d4cbdc0c9d02dcd97a83e0d Mon Sep 17 00:00:00 2001 From: AstrakhantsevaAA Date: Tue, 19 Sep 2023 11:45:16 +0200 Subject: [PATCH] fixed code strings --- docs/pydoc-markdown.yml | 4 +- .../docs/api_reference/__init__/__init__.md | 70 +++++++++---------- docs/website/docs/api_reference/dbt.md | 14 ++-- docs/website/docs/api_reference/helpers.md | 10 +-- docs/website/docs/api_reference/pipeline.md | 64 ++++++++--------- docs/website/docs/api_reference/sidebar.json | 2 +- docs/website/docs/api_reference/trace.md | 2 +- 7 files changed, 84 insertions(+), 82 deletions(-) diff --git a/docs/pydoc-markdown.yml b/docs/pydoc-markdown.yml index 4dbc053ad6..e25ba6278d 100644 --- a/docs/pydoc-markdown.yml +++ b/docs/pydoc-markdown.yml @@ -12,4 +12,6 @@ renderer: relative_output_path: api_reference relative_sidebar_path: sidebar.json sidebar_top_level_label: dlt - sidebar_top_level_module_label: null \ No newline at end of file + sidebar_top_level_module_label: pipeline + markdown: + escape_html_in_docstring: false \ No newline at end of file diff --git a/docs/website/docs/api_reference/__init__/__init__.md b/docs/website/docs/api_reference/__init__/__init__.md index 4eaadec036..7c938c2b3c 100644 --- a/docs/website/docs/api_reference/__init__/__init__.md +++ b/docs/website/docs/api_reference/__init__/__init__.md @@ -36,33 +36,33 @@ Please refer to the following doc pages: - `pipeline_name` _str, optional_ - A name of the pipeline that will be used to identify it in monitoring events and to restore its state and data schemas on subsequent runs. Defaults to the file name of a pipeline script with `dlt_` prefix added. -- `pipelines_dir` _str, optional_ - A working directory in which pipeline state and temporary files will be stored. Defaults to user home directory: `pipeline`0. +- `pipelines_dir` _str, optional_ - A working directory in which pipeline state and temporary files will be stored. Defaults to user home directory: `~/dlt/pipelines/`. -- `pipeline`1 _TSecretValue, optional_ - A random value used for deterministic hashing during data anonymization. Defaults to a value derived from the pipeline name. +- `pipeline_salt` _TSecretValue, optional_ - A random value used for deterministic hashing during data anonymization. Defaults to a value derived from the pipeline name. Default value should not be used for any cryptographic purposes. -- `pipeline`2 _str | DestinationReference, optional_ - A name of the destination to which dlt will load the data, or a destination module imported from `pipeline`3. +- `destination` _str | DestinationReference, optional_ - A name of the destination to which dlt will load the data, or a destination module imported from `dlt.destination`. May also be provided to `run` method of the `pipeline`. -- `pipeline`6 _str | DestinationReference, optional_ - A name of the destination where dlt will stage the data before final loading, or a destination module imported from `pipeline`3. +- `staging` _str | DestinationReference, optional_ - A name of the destination where dlt will stage the data before final loading, or a destination module imported from `dlt.destination`. May also be provided to `run` method of the `pipeline`. -- `Pipeline`0 _str, optional_ - A name of the dataset to which the data will be loaded. A dataset is a logical group of tables i.e. `Pipeline`1 in relational databases or folder grouping many files. +- `dataset_name` _str, optional_ - A name of the dataset to which the data will be loaded. A dataset is a logical group of tables i.e. `schema` in relational databases or folder grouping many files. May also be provided later to the `run` or `load` methods of the `Pipeline`. If not provided at all, then default to the `pipeline_name` -- `Pipeline`6 _str, optional_ - A path from which the schema `Pipeline`7 file will be imported on each pipeline run. Defaults to None which disables importing. +- `import_schema_path` _str, optional_ - A path from which the schema `yaml` file will be imported on each pipeline run. Defaults to None which disables importing. -- `Pipeline`8 _str, optional_ - A path where the schema `Pipeline`7 file will be exported after every schema change. Defaults to None which disables exporting. +- `export_schema_path` _str, optional_ - A path where the schema `yaml` file will be exported after every schema change. Defaults to None which disables exporting. -- `run`0 _bool, optional_ - When set to True, each instance of the pipeline with the `pipeline_name` starts from scratch when run and loads the data to a separate dataset. - The datasets are identified by `run`2 + datetime suffix. Use this setting whenever you experiment with your data to be sure you start fresh on each run. Defaults to False. +- `full_refresh` _bool, optional_ - When set to True, each instance of the pipeline with the `pipeline_name` starts from scratch when run and loads the data to a separate dataset. + The datasets are identified by `dataset_name_` + datetime suffix. Use this setting whenever you experiment with your data to be sure you start fresh on each run. Defaults to False. -- `run`3 _Any, optional_ - Credentials for the `pipeline`2 i.e. database connection string or a dictionary with Google cloud credentials. - In most cases should be set to None, which lets `dlt` to use `run`6 or environment variables to infer the right credentials values. +- `credentials` _Any, optional_ - Credentials for the `destination` i.e. database connection string or a dictionary with Google cloud credentials. + In most cases should be set to None, which lets `dlt` to use `secrets.toml` or environment variables to infer the right credentials values. -- `run`7 _str, Collector_ - A progress monitor that shows progress bars, console or log messages with current information on sources, resources, data items etc. processed in - `extract`, `normalize` and `load` stage. Pass a string with a collector name or configure your own by choosing from `extract`1 module. - We support most of the progress libraries: try passing `extract`2, `extract`3 or `extract`4 or `extract`5 to write to console/log. +- `progress` _str, Collector_ - A progress monitor that shows progress bars, console or log messages with current information on sources, resources, data items etc. processed in + `extract`, `normalize` and `load` stage. Pass a string with a collector name or configure your own by choosing from `dlt.progress` module. + We support most of the progress libraries: try passing `tqdm`, `enlighten` or `alive_progress` or `log` to write to console/log. **Returns**: @@ -115,43 +115,43 @@ Summary: This method will `extract` the data from the `data` argument, infer the schema, `normalize` the data into a load package (i.e. jsonl or PARQUET files representing tables) and then `load` such packages into the `destination`. The data may be supplied in several forms: -- a `list` or `Iterable` of any JSON-serializable objects i.e. `destination`0 -- any `destination`1 or a function that yield (`destination`2) i.e. `destination`3 -- a function or a list of functions decorated with @dlt.resource i.e. `destination`4 +- a `list` or `Iterable` of any JSON-serializable objects i.e. `dlt.run([1, 2, 3], table_name="numbers")` +- any `Iterator` or a function that yield (`Generator`) i.e. `dlt.run(range(1, 10), table_name="range")` +- a function or a list of functions decorated with @dlt.resource i.e. `dlt.run([chess_players(title="GM"), chess_games()])` - a function or a list of functions decorated with @dlt.source. -Please note that `destination`5 deals with `destination`6, `destination`7, `destination`8 and `destination`9 objects, so you are free to load binary data or documents containing dates. +Please note that `dlt` deals with `bytes`, `datetime`, `decimal` and `uuid` objects, so you are free to load binary data or documents containing dates. Execution: -The `dataset_name`0 method will first use `dataset_name`1 method to synchronize pipeline state and schemas with the destination. You can disable this behavior with `dataset_name`2 configuration option. -Next, it will make sure that data from the previous is fully processed. If not, `dataset_name`0 method normalizes and loads pending data items. +The `run` method will first use `sync_destination` method to synchronize pipeline state and schemas with the destination. You can disable this behavior with `restore_from_destination` configuration option. +Next, it will make sure that data from the previous is fully processed. If not, `run` method normalizes and loads pending data items. Only then the new data from `data` argument is extracted, normalized and loaded. **Arguments**: - `data` _Any_ - Data to be loaded to destination. -- `destination` _str | DestinationReference, optional_ - A name of the destination to which dlt will load the data, or a destination module imported from `dataset_name`7. - If not provided, the value passed to `dataset_name`8 will be used. +- `destination` _str | DestinationReference, optional_ - A name of the destination to which dlt will load the data, or a destination module imported from `dlt.destination`. + If not provided, the value passed to `dlt.pipeline` will be used. -- `dataset_name` _str, optional_ - A name of the dataset to which the data will be loaded. A dataset is a logical group of tables i.e. `extract`0 in relational databases or folder grouping many files. - If not provided, the value passed to `dataset_name`8 will be used. If not provided at all, then default to the `extract`2 +- `dataset_name` _str, optional_ - A name of the dataset to which the data will be loaded. A dataset is a logical group of tables i.e. `schema` in relational databases or folder grouping many files. + If not provided, the value passed to `dlt.pipeline` will be used. If not provided at all, then default to the `pipeline_name` -- `extract`3 _Any, optional_ - Credentials for the `destination` i.e. database connection string or a dictionary with Google cloud credentials. - In most cases should be set to None, which lets `destination`5 to use `extract`6 or environment variables to infer the right credentials values. +- `credentials` _Any, optional_ - Credentials for the `destination` i.e. database connection string or a dictionary with Google cloud credentials. + In most cases should be set to None, which lets `dlt` to use `secrets.toml` or environment variables to infer the right credentials values. -- `extract`7 _str, optional_ - The name of the table to which the data should be loaded within the `extract`8. This argument is required for a `data` that is a list/Iterable or Iterator without `data`0 attribute. +- `table_name` _str, optional_ - The name of the table to which the data should be loaded within the `dataset`. This argument is required for a `data` that is a list/Iterable or Iterator without `__name__` attribute. The behavior of this argument depends on the type of the `data`: - * generator functions: the function name is used as table name, `extract`7 overrides this default - * `data`3: resource contains the full table schema, and that includes the table name. `extract`7 will override this property. Use with care! - * `data`5: source contains several resources each with a table schema. `extract`7 will override all table names within the source and load the data into a single table. + * generator functions: the function name is used as table name, `table_name` overrides this default + * `@dlt.resource`: resource contains the full table schema, and that includes the table name. `table_name` will override this property. Use with care! + * `@dlt.source`: source contains several resources each with a table schema. `table_name` will override all table names within the source and load the data into a single table. -- `data`7 _Literal["skip", "append", "replace", "merge"], optional_ - Controls how to write data to a table. `data`8 will always add new data at the end of the table. `data`9 will replace existing data with new data. `normalize`0 will prevent data from loading. "merge" will deduplicate and merge data based on "primary_key" and "merge_key" hints. Defaults to "append". - Please note that in case of `normalize`1 the table schema value will be overwritten and in case of `normalize`2, the values in all resources will be overwritten. +- `write_disposition` _Literal["skip", "append", "replace", "merge"], optional_ - Controls how to write data to a table. `append` will always add new data at the end of the table. `replace` will replace existing data with new data. `skip` will prevent data from loading. "merge" will deduplicate and merge data based on "primary_key" and "merge_key" hints. Defaults to "append". + Please note that in case of `dlt.resource` the table schema value will be overwritten and in case of `dlt.source`, the values in all resources will be overwritten. -- `normalize`3 _Sequence[TColumnSchema], optional_ - A list of column schemas. Typed dictionary describing column names, data types, write disposition and performance hints that gives you full control over the created table schema. +- `columns` _Sequence[TColumnSchema], optional_ - A list of column schemas. Typed dictionary describing column names, data types, write disposition and performance hints that gives you full control over the created table schema. -- `extract`0 _Schema, optional_ - An explicit `normalize`5 object in which all table schemas will be grouped. By default, `destination`5 takes the schema from the source (if passed in `data` argument) or creates a default one itself. +- `schema` _Schema, optional_ - An explicit `Schema` object in which all table schemas will be grouped. By default, `dlt` takes the schema from the source (if passed in `data` argument) or creates a default one itself. **Raises**: @@ -160,5 +160,5 @@ Only then the new data from `data` argument is extracted, normalized and loaded. **Returns**: -- `load`1 - Information on loaded data including the list of package ids and failed job statuses. Please note that `destination`5 will not raise if a single job terminally fails. Such information is provided via LoadInfo. +- `LoadInfo` - Information on loaded data including the list of package ids and failed job statuses. Please note that `dlt` will not raise if a single job terminally fails. Such information is provided via LoadInfo. diff --git a/docs/website/docs/api_reference/dbt.md b/docs/website/docs/api_reference/dbt.md index fe4169138a..79b5ae1825 100644 --- a/docs/website/docs/api_reference/dbt.md +++ b/docs/website/docs/api_reference/dbt.md @@ -22,8 +22,8 @@ as required by `pipeline`. - `pipeline` _Pipeline_ - A pipeline for which the required dbt dependencies are inferred - `venv_path` _str, optional_ - A path where virtual environment is created or restored from. - If relative path is provided, the environment will be created within pipeline's working directory. Defaults to "dbt". -- `dbt_version` _str, optional_ - Version of dbt to be used. Exact version (ie. "1.2.4") or pip requirements string (ie. ">=1.1<1.5" may be provided). + If relative path is provided, the environment will be created within pipeline's working directory. Defaults to "dbt". +- `dbt_version` _str, optional_ - Version of dbt to be used. Exact version (ie. "1.2.4") or pip requirements string (ie. ">=1.1<1.5" may be provided). **Returns**: @@ -54,14 +54,14 @@ The wrapper creates a `dbt` profile from `dlt` pipeline configuration. Specifica - `pipeline` _Pipeline_ - A pipeline containing destination, credentials and dataset_name used to configure the dbt package. - `package_location` _str_ - A git repository url to be cloned or a local path where dbt package is present -- `dbt`0 _str, optional_ - A branch name, tag name or commit-id to check out. Defaults to None. -- `dbt`1 _TSecretValue, optional_ - SSH key to be used to clone private repositories. Defaults to TSecretValue(""). -- `dbt`2 _bool, optional_ - If set to True (default), the wrapper will automatically fall back to full-refresh mode when schema is out of sync -- `dbt`3 - https://docs.getdbt.com/docs/build/incremental-models#what-if-the-columns-of-my-incremental-model-change +- `package_repository_branch` _str, optional_ - A branch name, tag name or commit-id to check out. Defaults to None. +- `package_repository_ssh_key` _TSecretValue, optional_ - SSH key to be used to clone private repositories. Defaults to TSecretValue(""). +- `auto_full_refresh_when_out_of_sync` _bool, optional_ - If set to True (default), the wrapper will automatically fall back to full-refresh mode when schema is out of sync +- `See` - https://docs.getdbt.com/docs/build/incremental-models#what-if-the-columns-of-my-incremental-model-change - `venv` _Venv, optional_ - A virtual environment with required dbt dependencies. Defaults to None which will execute dbt package in current environment. **Returns**: -- `dbt`5 - A configured and authenticated Python `dbt` wrapper +- `DBTPackageRunner` - A configured and authenticated Python `dbt` wrapper diff --git a/docs/website/docs/api_reference/helpers.md b/docs/website/docs/api_reference/helpers.md index fe8f65bf4a..44c53412cd 100644 --- a/docs/website/docs/api_reference/helpers.md +++ b/docs/website/docs/api_reference/helpers.md @@ -15,12 +15,12 @@ A retry strategy for Tenacity that, with default setting, will repeat `load` ste Use this condition with tenacity `retry_if_exception`. Terminal exceptions are exceptions that will not go away when operations is repeated. Examples: missing configuration values, Authentication Errors, terminally failed jobs exceptions etc. ->>> data = source(...) ->>> for attempt in Retrying(stop=stop_after_attempt(3), retry=retry_if_exception(retry_load(())), reraise=True): ->>> with attempt: ->>> p.run(data) +>>> data = source(...) +>>> for attempt in Retrying(stop=stop_after_attempt(3), retry=retry_if_exception(retry_load(())), reraise=True): +>>> with attempt: +>>> p.run(data) **Arguments**: -- `retry_on_pipeline_steps` _Tuple[TPipelineStep, ...], optional_ - which pipeline steps are allowed to be repeated. Default: "load" +- `retry_on_pipeline_steps` _Tuple[TPipelineStep, ...], optional_ - which pipeline steps are allowed to be repeated. Default: "load" diff --git a/docs/website/docs/api_reference/pipeline.md b/docs/website/docs/api_reference/pipeline.md index 58fc5e1276..aa7c094237 100644 --- a/docs/website/docs/api_reference/pipeline.md +++ b/docs/website/docs/api_reference/pipeline.md @@ -19,7 +19,7 @@ Indicates a first run of the pipeline, where run ends with successful loading of #### pipelines\_dir -A directory where the pipelines' working directories are created +A directory where the pipelines' working directories are created #### working\_dir @@ -78,7 +78,7 @@ def extract(data: Any, workers: int = None) -> ExtractInfo ``` -Extracts the `data` and prepare it for the normalization. Does not require destination or credentials to be configured. See `run` method for the arguments' description. +Extracts the `data` and prepare it for the normalization. Does not require destination or credentials to be configured. See `run` method for the arguments' description. #### normalize @@ -134,52 +134,52 @@ Loads the data from `data` argument into the destination specified in `destinati This method will `extract` the data from the `data` argument, infer the schema, `normalize` the data into a load package (ie. jsonl or PARQUET files representing tables) and then `load` such packages into the `destination`. The data may be supplied in several forms: -* a `list` or `Iterable` of any JSON-serializable objects ie. `destination`0 -* any `destination`1 or a function that yield (`destination`2) ie. `destination`3 -* a function or a list of functions decorated with @dlt.resource ie. `destination`4 +* a `list` or `Iterable` of any JSON-serializable objects ie. `dlt.run([1, 2, 3], table_name="numbers")` +* any `Iterator` or a function that yield (`Generator`) ie. `dlt.run(range(1, 10), table_name="range")` +* a function or a list of functions decorated with @dlt.resource ie. `dlt.run([chess_players(title="GM"), chess_games()])` * a function or a list of functions decorated with @dlt.source. -Please note that `destination`5 deals with `destination`6, `destination`7, `destination`8 and `destination`9 objects so you are free to load documents containing ie. binary data or dates. +Please note that `dlt` deals with `bytes`, `datetime`, `decimal` and `uuid` objects so you are free to load documents containing ie. binary data or dates. ### Execution -The `dataset_name`0 method will first use `dataset_name`1 method to synchronize pipeline state and schemas with the destination. You can disable this behavior with `dataset_name`2 configuration option. -Next it will make sure that data from the previous is fully processed. If not, `dataset_name`0 method normalizes, loads pending data items and **exits** +The `run` method will first use `sync_destination` method to synchronize pipeline state and schemas with the destination. You can disable this behavior with `restore_from_destination` configuration option. +Next it will make sure that data from the previous is fully processed. If not, `run` method normalizes, loads pending data items and **exits** If there was no pending data, new data from `data` argument is extracted, normalized and loaded. ### Args: data (Any): Data to be loaded to destination -destination (str | DestinationReference, optional): A name of the destination to which dlt will load the data, or a destination module imported from `dataset_name`5. -If not provided, the value passed to `dataset_name`6 will be used. +destination (str | DestinationReference, optional): A name of the destination to which dlt will load the data, or a destination module imported from `dlt.destination`. +If not provided, the value passed to `dlt.pipeline` will be used. -dataset_name (str, optional):A name of the dataset to which the data will be loaded. A dataset is a logical group of tables ie. `dataset_name`7 in relational databases or folder grouping many files. -If not provided, the value passed to `dataset_name`6 will be used. If not provided at all then defaults to the `dataset_name`9 +dataset_name (str, optional):A name of the dataset to which the data will be loaded. A dataset is a logical group of tables ie. `schema` in relational databases or folder grouping many files. +If not provided, the value passed to `dlt.pipeline` will be used. If not provided at all then defaults to the `pipeline_name` credentials (Any, optional): Credentials for the `destination` ie. database connection string or a dictionary with google cloud credentials. -In most cases should be set to None, which lets `destination`5 to use `extract`2 or environment variables to infer right credentials values. +In most cases should be set to None, which lets `dlt` to use `secrets.toml` or environment variables to infer right credentials values. -table_name (str, optional): The name of the table to which the data should be loaded within the `extract`3. This argument is required for a `data` that is a list/Iterable or Iterator without `extract`5 attribute. +table_name (str, optional): The name of the table to which the data should be loaded within the `dataset`. This argument is required for a `data` that is a list/Iterable or Iterator without `__name__` attribute. The behavior of this argument depends on the type of the `data`: -* generator functions: the function name is used as table name, `extract`7 overrides this default -* `extract`8: resource contains the full table schema and that includes the table name. `extract`7 will override this property. Use with care! -* `data`0: source contains several resources each with a table schema. `extract`7 will override all table names within the source and load the data into single table. +* generator functions: the function name is used as table name, `table_name` overrides this default +* `@dlt.resource`: resource contains the full table schema and that includes the table name. `table_name` will override this property. Use with care! +* `@dlt.source`: source contains several resources each with a table schema. `table_name` will override all table names within the source and load the data into single table. -write_disposition (Literal["skip", "append", "replace", "merge"], optional): Controls how to write data to a table. `data`2 will always add new data at the end of the table. `data`3 will replace existing data with new data. `data`4 will prevent data from loading. "merge" will deduplicate and merge data based on "primary_key" and "merge_key" hints. Defaults to "append". -Please note that in case of `data`5 the table schema value will be overwritten and in case of `data`6, the values in all resources will be overwritten. +write_disposition (Literal["skip", "append", "replace", "merge"], optional): Controls how to write data to a table. `append` will always add new data at the end of the table. `replace` will replace existing data with new data. `skip` will prevent data from loading. "merge" will deduplicate and merge data based on "primary_key" and "merge_key" hints. Defaults to "append". +Please note that in case of `dlt.resource` the table schema value will be overwritten and in case of `dlt.source`, the values in all resources will be overwritten. columns (Sequence[TColumnSchema], optional): A list of column schemas. Typed dictionary describing column names, data types, write disposition and performance hints that gives you full control over the created table schema. -primary_key (str | Sequence[str]): A column name or a list of column names that comprise a private key. Typically used with "merge" write disposition to deduplicate loaded data. +primary_key (str | Sequence[str]): A column name or a list of column names that comprise a private key. Typically used with "merge" write disposition to deduplicate loaded data. -schema (Schema, optional): An explicit `data`7 object in which all table schemas will be grouped. By default `destination`5 takes the schema from the source (if passed in `data` argument) or creates a default one itself. +schema (Schema, optional): An explicit `Schema` object in which all table schemas will be grouped. By default `dlt` takes the schema from the source (if passed in `data` argument) or creates a default one itself. -loader_file_format (Literal["jsonl", "insert_values", "parquet"], optional). The file format the loader will use to create the load package. Not all file_formats are compatible with all destinations. Defaults to the preferred file format of the selected destination. +loader_file_format (Literal["jsonl", "insert_values", "parquet"], optional). The file format the loader will use to create the load package. Not all file_formats are compatible with all destinations. Defaults to the preferred file format of the selected destination. ### Raises: PipelineStepFailed when a problem happened during `extract`, `normalize` or `load` steps. ### Returns: -LoadInfo: Information on loaded data including the list of package ids and failed job statuses. Please not that `destination`5 will not raise if a single job terminally fails. Such information is provided via LoadInfo. +LoadInfo: Information on loaded data including the list of package ids and failed job statuses. Please not that `dlt` will not raise if a single job terminally fails. Such information is provided via LoadInfo. #### sync\_destination @@ -190,7 +190,7 @@ def sync_destination(destination: TDestinationReferenceArg = None, dataset_name: str = None) -> None ``` -Synchronizes pipeline state with the `destination`'s state kept in `dataset_name` +Synchronizes pipeline state with the `destination`'s state kept in `dataset_name` ### Summary Attempts to restore pipeline state and schemas from the destination. Requires the state that is present at the destination to have a higher version number that state kept locally in working directory. @@ -221,7 +221,7 @@ and maximum identifier length. Only one pipeline is active at a given time. -Pipeline created or attached with `dlt.pipeline`/'dlt.attach`pipeline.run`0run`pipeline.run`1load`pipeline.run`2extract` methods also activate pipeline. +Pipeline created or attached with `dlt.pipeline`/'dlt.attach` is automatically activated. `run`, `load` and `extract` methods also activate pipeline. #### deactivate @@ -344,11 +344,11 @@ def sql_client(schema_name: str = None, Returns a sql client configured to query/change the destination and dataset that were used to load the data. Use the client with `with` statement to manage opening and closing connection to the destination: ->>> with pipeline.sql_client() as client: ->>> with client.execute_query( ->>> "SELECT id, name, email FROM customers WHERE id = %s", 10 ->>> ) as cursor: ->>> print(cursor.fetchall()) +>>> with pipeline.sql_client() as client: +>>> with client.execute_query( +>>> "SELECT id, name, email FROM customers WHERE id = %s", 10 +>>> ) as cursor: +>>> print(cursor.fetchall()) The client is authenticated and defaults all queries to dataset_name used by the pipeline. You can provide alternative `schema_name` which will be used to normalize dataset name and alternative `credentials`. @@ -362,8 +362,8 @@ def destination_client(schema_name: str = None, Get the destination job client for the configured destination Use the client with `with` statement to manage opening and closing connection to the destination: ->>> with pipeline.destination_client() as client: ->>> client.drop_storage() # removes storage which typically wipes all data in it +>>> with pipeline.destination_client() as client: +>>> client.drop_storage() # removes storage which typically wipes all data in it The client is authenticated. You can provide alternative `schema_name` which will be used to normalize dataset name and alternative `credentials`. If no schema name is provided and no default schema is present in the pipeline, and ad hoc schema will be created and discarded after use. diff --git a/docs/website/docs/api_reference/sidebar.json b/docs/website/docs/api_reference/sidebar.json index 77df68e2f4..39ea33ed66 100644 --- a/docs/website/docs/api_reference/sidebar.json +++ b/docs/website/docs/api_reference/sidebar.json @@ -15,6 +15,6 @@ "api_reference/trace", "api_reference/track" ], - "label": "dlt", + "label": "Reference", "type": "category" } \ No newline at end of file diff --git a/docs/website/docs/api_reference/trace.md b/docs/website/docs/api_reference/trace.md index 46ac2522c0..a357174d22 100644 --- a/docs/website/docs/api_reference/trace.md +++ b/docs/website/docs/api_reference/trace.md @@ -57,7 +57,7 @@ A dictionary representation of PipelineStepTrace that can be loaded with `dlt` class PipelineTrace() ``` -Pipeline runtime trace containing data on "extract", "normalize" and "load" steps and resolved config and secret values. +Pipeline runtime trace containing data on "extract", "normalize" and "load" steps and resolved config and secret values. #### steps