Skip to content

Commit

Permalink
Init docstrings for destination factories
Browse files Browse the repository at this point in the history
  • Loading branch information
steinitzu committed Nov 16, 2023
1 parent 481a7cb commit d91402c
Show file tree
Hide file tree
Showing 9 changed files with 110 additions and 3 deletions.
14 changes: 12 additions & 2 deletions dlt/destinations/impl/athena/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,26 @@ def __init__(
credentials: t.Union[AwsCredentials, t.Dict[str, t.Any], t.Any] = None,
athena_work_group: t.Optional[str] = None,
aws_data_catalog: t.Optional[str] = "awsdatacatalog",
supports_truncate_command: bool = False,
force_iceberg: bool = False,
**kwargs: t.Any,
) -> None:
"""Configure the Athena destination to use in a pipeline.
All arguments provided here supersede other configuration sources such as environment variables and dlt config files.
Args:
query_result_bucket: S3 bucket to store query results in
credentials: AWS credentials to connect to the Athena database.
athena_work_group: Athena work group to use
aws_data_catalog: Athena data catalog to use
force_iceberg: Force iceberg tables
**kwargs: Additional arguments passed to the destination config
"""
super().__init__(
query_result_bucket=query_result_bucket,
credentials=credentials,
athena_work_group=athena_work_group,
aws_data_catalog=aws_data_catalog,
supports_truncate_command=supports_truncate_command,
force_iceberg=force_iceberg,
**kwargs,
)
10 changes: 10 additions & 0 deletions dlt/destinations/impl/duckdb/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,14 @@ def __init__(
create_indexes: bool = False,
**kwargs: t.Any,
) -> None:
"""Configure the DuckDB destination to use in a pipeline.
All arguments provided here supersede other configuration sources such as environment variables and dlt config files.
Args:
credentials: Credentials to connect to the duckdb database. Can be an instance of `DuckDbCredentials` or
a path to a database file. Use `:memory:` to create an in-memory database.
create_indexes: Should unique indexes be created
**kwargs: Additional arguments passed to the destination config
"""
super().__init__(credentials=credentials, create_indexes=create_indexes, **kwargs)
19 changes: 19 additions & 0 deletions dlt/destinations/impl/filesystem/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,23 @@ def __init__(
credentials: t.Union[FileSystemCredentials, t.Dict[str, t.Any], t.Any] = None,
**kwargs: t.Any,
) -> None:
"""Configure the filesystem destination to use in a pipeline and load data to local or remote filesystem.
All arguments provided here supersede other configuration sources such as environment variables and dlt config files.
The `bucket_url` determines the protocol to be used:
- Local folder: `file:///path/to/directory`
- AWS S3 (and S3 compatible storages): `s3://bucket-name
- Azure Blob Storage: `az://container-name
- Google Cloud Storage: `gs://bucket-name
- Memory fs: `memory://m`
Args:
bucket_url: The fsspec compatible bucket url to use for the destination.
credentials: Credentials to connect to the filesystem. The type of credentials should correspond to
the bucket protocol. For example, for AWS S3, the credentials should be an instance of `AwsCredentials`.
A dictionary with the credentials parameters can also be provided.
**kwargs: Additional arguments passed to the destination config
"""
super().__init__(bucket_url=bucket_url, credentials=credentials, **kwargs)
10 changes: 10 additions & 0 deletions dlt/destinations/impl/motherduck/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,14 @@ def __init__(
create_indexes: bool = False,
**kwargs: t.Any,
) -> None:
"""Configure the MotherDuck destination to use in a pipeline.
All arguments provided here supersede other configuration sources such as environment variables and dlt config files.
Args:
credentials: Credentials to connect to the MotherDuck database. Can be an instance of `MotherDuckCredentials` or
a connection string in the format `md:///<database_name>?token=<service token>`
create_indexes: Should unique indexes be created
**kwargs: Additional arguments passed to the destination config
"""
super().__init__(credentials=credentials, create_indexes=create_indexes, **kwargs)
10 changes: 10 additions & 0 deletions dlt/destinations/impl/mssql/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,14 @@ def __init__(
create_indexes: bool = True,
**kwargs: t.Any,
) -> None:
"""Configure the MsSql destination to use in a pipeline.
All arguments provided here supersede other configuration sources such as environment variables and dlt config files.
Args:
credentials: Credentials to connect to the mssql database. Can be an instance of `MsSqlCredentials` or
a connection string in the format `mssql://user:password@host:port/database`
create_indexes: Should unique indexes be created
**kwargs: Additional arguments passed to the destination config
"""
super().__init__(credentials=credentials, create_indexes=create_indexes, **kwargs)
10 changes: 10 additions & 0 deletions dlt/destinations/impl/postgres/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,14 @@ def __init__(
create_indexes: bool = True,
**kwargs: t.Any,
) -> None:
"""Configure the Postgres destination to use in a pipeline.
All arguments provided here supersede other configuration sources such as environment variables and dlt config files.
Args:
credentials: Credentials to connect to the postgres database. Can be an instance of `PostgresCredentials` or
a connection string in the format `postgres://user:password@host:port/database`
create_indexes: Should unique indexes be created
**kwargs: Additional arguments passed to the destination config
"""
super().__init__(credentials=credentials, create_indexes=create_indexes, **kwargs)
11 changes: 11 additions & 0 deletions dlt/destinations/impl/redshift/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,17 @@ def __init__(
staging_iam_role: t.Optional[str] = None,
**kwargs: t.Any,
) -> None:
"""Configure the Redshift destination to use in a pipeline.
All arguments provided here supersede other configuration sources such as environment variables and dlt config files.
Args:
credentials: Credentials to connect to the redshift database. Can be an instance of `RedshiftCredentials` or
a connection string in the format `redshift://user:password@host:port/database`
create_indexes: Should unique indexes be created
staging_iam_role: IAM role to use for staging data in S3
**kwargs: Additional arguments passed to the destination config
"""
super().__init__(
credentials=credentials, create_indexes=create_indexes, staging_iam_role=staging_iam_role, **kwargs
)
10 changes: 10 additions & 0 deletions dlt/destinations/impl/snowflake/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,14 @@ def __init__(
keep_staged_files: bool = True,
**kwargs: t.Any,
) -> None:
"""Configure the Snowflake destination to use in a pipeline.
All arguments provided here supersede other configuration sources such as environment variables and dlt config files.
Args:
credentials: Credentials to connect to the snowflake database. Can be an instance of `SnowflakeCredentials` or
a connection string in the format `snowflake://user:password@host:port/database`
stage_name: Name of an existing stage to use for loading data. Default uses implicit stage per table
keep_staged_files: Whether to delete or keep staged files after loading
"""
super().__init__(credentials=credentials, stage_name=stage_name, keep_staged_files=keep_staged_files, **kwargs)
19 changes: 18 additions & 1 deletion dlt/destinations/impl/weaviate/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,23 @@ def client_class(self) -> t.Type["WeaviateClient"]:
def __init__(
self,
credentials: t.Union[WeaviateCredentials, t.Dict[str, t.Any]] = None,
vectorizer: str = None,
module_config: t.Dict[str, t.Dict[str, str]] = None,
**kwargs: t.Any,
) -> None:
super().__init__(credentials=credentials, **kwargs)
"""Configure the Weaviate destination to use in a pipeline.
All destination config parameters can be provided as arguments here and will supersede other config sources (such as dlt config files and environment variables).
Args:
credentials: Weaviate credentials containing URL, API key and optional headers
vectorizer: The name of the Weaviate vectorizer to use
module_config: The configuration for the Weaviate modules
**kwargs: Additional arguments forwarded to the destination config
"""
super().__init__(
credentials=credentials,
vectorizer=vectorizer,
module_config=module_config,
**kwargs
)

0 comments on commit d91402c

Please sign in to comment.