diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index fc7eeadfe2..ada73b85d9 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -30,6 +30,7 @@ env: # postgres runs again here so we can test on mac/windows ACTIVE_DESTINATIONS: "[\"redshift\", \"postgres\", \"duckdb\", \"filesystem\", \"dummy\"]" # note that all buckets are enabled for testing + ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\", \"r2\", \"s3\", \"gs\", \"az\", \"abfss\", \"gdrive\"]" #excludes sftp jobs: get_docs_changes: diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml index a21c3f0618..8911e05ecc 100644 --- a/.github/workflows/test_local_destinations.yml +++ b/.github/workflows/test_local_destinations.yml @@ -22,7 +22,7 @@ env: RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} ACTIVE_DESTINATIONS: "[\"duckdb\", \"postgres\", \"filesystem\", \"weaviate\", \"qdrant\"]" - ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\"]" + ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\", \"sftp\"]" DESTINATION__WEAVIATE__VECTORIZER: text2vec-contextionary DESTINATION__WEAVIATE__MODULE_CONFIG: "{\"text2vec-contextionary\": {\"vectorizeClassName\": false, \"vectorizePropertyName\": true}}" @@ -95,16 +95,35 @@ jobs: key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-local-destinations - name: Install dependencies - run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate -E qdrant --with sentry-sdk --with pipeline -E deltalake + run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate -E qdrant -E sftp --with sentry-sdk --with pipeline -E deltalake + + - name: Start SFTP server + run: docker compose -f "tests/load/filesystem_sftp/docker-compose.yml" up -d + + - name: Configure SSH Agent for sftp tests + run: | + mkdir -p /home/runner/.ssh + cp tests/load/filesystem_sftp/bootstrap/bobby_rsa /home/runner/.ssh/id_rsa + cp tests/load/filesystem_sftp/bootstrap/bobby_rsa.pub /home/runner/.ssh/id_rsa.pub # always run full suite, also on branches - - run: poetry run pytest tests/load --ignore tests/load/sources && poetry run pytest tests/cli - name: Run tests Linux + - name: Run tests Linux + run: | + eval "$(ssh-agent -s)" + poetry run pytest tests/load --ignore tests/load/sources + poetry run pytest tests/cli env: DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader:loader@localhost:5432/dlt_data DESTINATION__QDRANT__CREDENTIALS__location: http://localhost:6333 + DESTINATION__FILESYSTEM__CREDENTIALS__SFTP_PORT: 2222 + DESTINATION__FILESYSTEM__CREDENTIALS__SFTP_USERNAME: foo + DESTINATION__FILESYSTEM__CREDENTIALS__SFTP_PASSWORD: pass - name: Stop weaviate if: always() run: docker compose -f ".github/weaviate-compose.yml" down -v + + - name: Stop SFTP server + if: always() + run: docker compose -f "tests/load/filesystem_sftp/docker-compose.yml" down -v diff --git a/.github/workflows/test_pyarrow17.yml b/.github/workflows/test_pyarrow17.yml index dd48c2af9d..78d6742ac1 100644 --- a/.github/workflows/test_pyarrow17.yml +++ b/.github/workflows/test_pyarrow17.yml @@ -23,6 +23,7 @@ env: RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} ACTIVE_DESTINATIONS: "[\"filesystem\"]" + ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\", \"r2\", \"s3\", \"gs\", \"az\", \"abfss\", \"gdrive\"]" #excludes sftp jobs: get_docs_changes: @@ -72,6 +73,7 @@ jobs: - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - - run: | - poetry run pytest tests/libs tests/load -m needspyarrow17 - name: Run needspyarrow17 tests Linux + - name: Run needspyarrow17 tests Linux + run: | + poetry run pytest tests/libs -m "needspyarrow17" + poetry run pytest tests/load -m "needspyarrow17" diff --git a/dlt/common/configuration/specs/__init__.py b/dlt/common/configuration/specs/__init__.py index f1d7d819ff..179445dde3 100644 --- a/dlt/common/configuration/specs/__init__.py +++ b/dlt/common/configuration/specs/__init__.py @@ -28,6 +28,7 @@ AnyAzureCredentials, ) +from .sftp_crendentials import SFTPCredentials # backward compatibility for service account credentials from .gcp_credentials import ( @@ -62,4 +63,5 @@ "AnyAzureCredentials", "GcpClientCredentials", "GcpClientCredentialsWithDefault", + "SFTPCredentials", ] diff --git a/dlt/common/configuration/specs/sftp_crendentials.py b/dlt/common/configuration/specs/sftp_crendentials.py new file mode 100644 index 0000000000..92f83cc438 --- /dev/null +++ b/dlt/common/configuration/specs/sftp_crendentials.py @@ -0,0 +1,69 @@ +from typing import Any, Dict, Optional + +from dlt.common.typing import TSecretStrValue, DictStrAny +from dlt.common.configuration.specs.base_configuration import CredentialsConfiguration, configspec + + +@configspec +class SFTPCredentials(CredentialsConfiguration): + """Credentials for SFTP filesystem, compatible with fsspec SFTP protocol. + + Authentication is attempted in the following order of priority: + + - `key_filename` may contain OpenSSH public certificate paths + as well as regular private-key paths; when files ending in `-cert.pub` are found, they are assumed to match + a private key, and both components will be loaded. + + - Any key found through an SSH agent: any “id_rsa”, “id_dsa”, or “id_ecdsa” key discoverable in ~/.ssh/. + + - Plain username/password authentication, if a password was provided. + + - If a private key requires a password to unlock it, and a password is provided, that password will be used to + attempt to unlock the key. + + For more information about parameters: + https://docs.paramiko.org/en/3.3/api/client.html#paramiko.client.SSHClient.connect + """ + + sftp_port: Optional[int] = 22 + sftp_username: Optional[str] = None + sftp_password: Optional[TSecretStrValue] = None + sftp_key_filename: Optional[str] = None + sftp_key_passphrase: Optional[TSecretStrValue] = None + sftp_timeout: Optional[float] = None + sftp_banner_timeout: Optional[float] = None + sftp_auth_timeout: Optional[float] = None + sftp_channel_timeout: Optional[float] = None + sftp_allow_agent: Optional[bool] = True + sftp_look_for_keys: Optional[bool] = True + sftp_compress: Optional[bool] = False + sftp_gss_auth: Optional[bool] = False + sftp_gss_kex: Optional[bool] = False + sftp_gss_deleg_creds: Optional[bool] = True + sftp_gss_host: Optional[str] = None + sftp_gss_trust_dns: Optional[bool] = True + + def to_fsspec_credentials(self) -> Dict[str, Any]: + """Return a dict that can be passed to fsspec SFTP/SSHClient.connect method.""" + + credentials: Dict[str, Any] = { + "port": self.sftp_port, + "username": self.sftp_username, + "password": self.sftp_password, + "key_filename": self.sftp_key_filename, + "passphrase": self.sftp_key_passphrase, + "timeout": self.sftp_timeout, + "banner_timeout": self.sftp_banner_timeout, + "auth_timeout": self.sftp_auth_timeout, + "channel_timeout": self.sftp_channel_timeout, + "allow_agent": self.sftp_allow_agent, + "look_for_keys": self.sftp_look_for_keys, + "compress": self.sftp_compress, + "gss_auth": self.sftp_gss_auth, + "gss_kex": self.sftp_gss_kex, + "gss_deleg_creds": self.sftp_gss_deleg_creds, + "gss_host": self.sftp_gss_host, + "gss_trust_dns": self.sftp_gss_trust_dns, + } + + return credentials diff --git a/dlt/common/libs/deltalake.py b/dlt/common/libs/deltalake.py index 48c4f33d9b..9caba55183 100644 --- a/dlt/common/libs/deltalake.py +++ b/dlt/common/libs/deltalake.py @@ -175,7 +175,7 @@ def get_delta_tables( def _deltalake_storage_options(config: FilesystemConfiguration) -> Dict[str, str]: """Returns dict that can be passed as `storage_options` in `deltalake` library.""" - creds = {} + creds = {} # type: ignore extra_options = {} # TODO: create a mixin with to_object_store_rs_credentials for a proper discovery if hasattr(config.credentials, "to_object_store_rs_credentials"): diff --git a/dlt/common/storages/configuration.py b/dlt/common/storages/configuration.py index 04780528c4..4da44bceee 100644 --- a/dlt/common/storages/configuration.py +++ b/dlt/common/storages/configuration.py @@ -12,6 +12,7 @@ GcpOAuthCredentials, AnyAzureCredentials, BaseConfiguration, + SFTPCredentials, ) from dlt.common.typing import DictStrAny from dlt.common.utils import digest128 @@ -48,10 +49,19 @@ class LoadStorageConfiguration(BaseConfiguration): FileSystemCredentials = Union[ - AwsCredentials, GcpServiceAccountCredentials, AnyAzureCredentials, GcpOAuthCredentials + AwsCredentials, + GcpServiceAccountCredentials, + AnyAzureCredentials, + GcpOAuthCredentials, + SFTPCredentials, ] +def _make_sftp_url(scheme: str, fs_path: str, bucket_url: str) -> str: + parsed_bucket_url = urlparse(bucket_url) + return f"{scheme}://{parsed_bucket_url.hostname}{fs_path}" + + def _make_az_url(scheme: str, fs_path: str, bucket_url: str) -> str: parsed_bucket_url = urlparse(bucket_url) if parsed_bucket_url.username: @@ -76,7 +86,7 @@ def _make_file_url(scheme: str, fs_path: str, bucket_url: str) -> str: return p_.as_uri() -MAKE_URI_DISPATCH = {"az": _make_az_url, "file": _make_file_url} +MAKE_URI_DISPATCH = {"az": _make_az_url, "file": _make_file_url, "sftp": _make_sftp_url} MAKE_URI_DISPATCH["adl"] = MAKE_URI_DISPATCH["az"] MAKE_URI_DISPATCH["abfs"] = MAKE_URI_DISPATCH["az"] @@ -109,6 +119,7 @@ class FilesystemConfiguration(BaseConfiguration): * az, abfs, adl, abfss, azure * file, memory * gdrive + * sftp """ PROTOCOL_CREDENTIALS: ClassVar[Dict[str, Any]] = { @@ -121,6 +132,7 @@ class FilesystemConfiguration(BaseConfiguration): "adl": AnyAzureCredentials, "abfss": AnyAzureCredentials, "azure": AnyAzureCredentials, + "sftp": SFTPCredentials, } bucket_url: str = None diff --git a/dlt/common/storages/fsspec_filesystem.py b/dlt/common/storages/fsspec_filesystem.py index 76dbd98e6e..6ac5f31007 100644 --- a/dlt/common/storages/fsspec_filesystem.py +++ b/dlt/common/storages/fsspec_filesystem.py @@ -30,6 +30,7 @@ GcpCredentials, AwsCredentials, AzureCredentials, + SFTPCredentials, ) from dlt.common.exceptions import MissingDependencyException from dlt.common.storages.configuration import ( @@ -64,6 +65,7 @@ class FileItem(TypedDict, total=False): "file": lambda f: ensure_pendulum_datetime(f["mtime"]), "memory": lambda f: ensure_pendulum_datetime(f["created"]), "gdrive": lambda f: ensure_pendulum_datetime(f["modifiedTime"]), + "sftp": lambda f: ensure_pendulum_datetime(f["mtime"]), } # Support aliases MTIME_DISPATCH["gs"] = MTIME_DISPATCH["gcs"] @@ -77,6 +79,7 @@ class FileItem(TypedDict, total=False): "az": lambda config: cast(AzureCredentials, config.credentials).to_adlfs_credentials(), "gs": lambda config: cast(GcpCredentials, config.credentials).to_gcs_credentials(), "gdrive": lambda config: {"credentials": cast(GcpCredentials, config.credentials)}, + "sftp": lambda config: cast(SFTPCredentials, config.credentials).to_fsspec_credentials(), } CREDENTIALS_DISPATCH["adl"] = CREDENTIALS_DISPATCH["az"] CREDENTIALS_DISPATCH["abfs"] = CREDENTIALS_DISPATCH["az"] @@ -136,6 +139,10 @@ def prepare_fsspec_args(config: FilesystemConfiguration) -> DictStrAny: register_implementation("gdrive", GoogleDriveFileSystem, "GoogleDriveFileSystem") fs_kwargs.update(DEFAULT_KWARGS.get(protocol, {})) + + if protocol == "sftp": + fs_kwargs.clear() + if config.kwargs is not None: fs_kwargs.update(config.kwargs) if config.client_kwargs is not None: @@ -155,6 +162,7 @@ def fsspec_from_config(config: FilesystemConfiguration) -> Tuple[AbstractFileSys * s3 * az, abfs, abfss, adl, azure * gcs, gs + * sftp All other filesystems are not authenticated diff --git a/docs/website/docs/dlt-ecosystem/destinations/filesystem.md b/docs/website/docs/dlt-ecosystem/destinations/filesystem.md index 675a3d6e8e..888cae7a35 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/filesystem.md +++ b/docs/website/docs/dlt-ecosystem/destinations/filesystem.md @@ -268,6 +268,98 @@ bucket_url='\\?\UNC\localhost\c$\a\b\c' ``` ::: +### SFTP +Run `pip install "dlt[sftp]` which will install the `paramiko` package alongside `dlt`, enabling secure SFTP transfers. + +Configure your SFTP credentials by editing the `.dlt/secrets.toml` file. By default, the file contains placeholders for AWS credentials. You should replace these with your SFTP credentials. + +Below are the possible fields for SFTP credentials configuration: + +```text +sftp_port # The port for SFTP, defaults to 22 (standard for SSH/SFTP) +sftp_username # Your SFTP username, defaults to None +sftp_password # Your SFTP password (if using password-based auth), defaults to None +sftp_key_filename # Path to your private key file for key-based authentication, defaults to None +sftp_key_passphrase # Passphrase for your private key (if applicable), defaults to None +sftp_timeout # Timeout for establishing a connection, defaults to None +sftp_banner_timeout # Timeout for receiving the banner during authentication, defaults to None +sftp_auth_timeout # Authentication timeout, defaults to None +sftp_channel_timeout # Channel timeout for SFTP operations, defaults to None +sftp_allow_agent # Use SSH agent for key management (if available), defaults to True +sftp_look_for_keys # Search for SSH keys in the default SSH directory (~/.ssh/), defaults to True +sftp_compress # Enable compression (can improve performance over slow networks), defaults to False +sftp_gss_auth # Use GSS-API for authentication, defaults to False +sftp_gss_kex # Use GSS-API for key exchange, defaults to False +sftp_gss_deleg_creds # Delegate credentials with GSS-API, defaults to True +sftp_gss_host # Host for GSS-API, defaults to None +sftp_gss_trust_dns # Trust DNS for GSS-API, defaults to True +``` +> For more information about credentials parameters: https://docs.paramiko.org/en/3.3/api/client.html#paramiko.client.SSHClient.connect + +### Authentication Methods + +SFTP authentication is attempted in the following order of priority: + +1. **Key-based authentication**: If you provide a `key_filename` containing the path to a private key or a corresponding OpenSSH public certificate (e.g., `id_rsa` and `id_rsa-cert.pub`), these will be used for authentication. If the private key requires a passphrase, you can specify it via `sftp_key_passphrase`. If your private key requires a passphrase to unlock, and you’ve provided one, it will be used to attempt to unlock the key. + +2. **SSH Agent-based authentication**: If `allow_agent=True` (default), Paramiko will look for any SSH keys stored in your local SSH agent (such as `id_rsa`, `id_dsa`, or `id_ecdsa` keys stored in `~/.ssh/`). + +3. **Username/Password authentication**: If a password is provided (`sftp_password`), plain username/password authentication will be attempted. + +4. **GSS-API authentication**: If GSS-API (Kerberos) is enabled (sftp_gss_auth=True), authentication will use the Kerberos protocol. GSS-API may also be used for key exchange (sftp_gss_kex=True) and credential delegation (sftp_gss_deleg_creds=True). This method is useful in environments where Kerberos is set up, often in enterprise networks. + + +#### 1. **Key-based Authentication** + +If you use an SSH key instead of a password, you can specify the path to your private key in the configuration. + +```toml +[destination.filesystem] +bucket_url = "sftp://[hostname]/[path]" +file_glob = "*" + +[destination.filesystem.credentials] +sftp_username = "foo" +sftp_key_filename = "/path/to/id_rsa" # Replace with the path to your private key file +sftp_key_passphrase = "your_passphrase" # Optional: passphrase for your private key +``` + +#### 2. **SSH Agent-based Authentication** + +If you have an SSH agent running with loaded keys, you can allow Paramiko to use these keys automatically. You can omit the password and key fields if you're relying on the SSH agent. + +```toml +[destination.filesystem] +bucket_url = "sftp://[hostname]/[path]" +file_glob = "*" + +[destination.filesystem.credentials] +sftp_username = "foo" +sftp_key_passphrase = "your_passphrase" # Optional: passphrase for your private key +``` +The loaded key must be one of the following types stored in ~/.ssh/: id_rsa, id_dsa, or id_ecdsa. + +#### 3. **Username/Password Authentication** + +This is the simplest form of authentication, where you supply a username and password directly. + +```toml +[destination.filesystem] +bucket_url = "sftp://[hostname]/[path]" # The hostname of your SFTP server and the remote path +file_glob = "*" # Pattern to match the files you want to upload/download + +[destination.filesystem.credentials] +sftp_username = "foo" # Replace "foo" with your SFTP username +sftp_password = "pass" # Replace "pass" with your SFTP password +``` + + +### Notes: +- **Key-based Authentication**: Make sure your private key has the correct permissions (`chmod 600`), or SSH will refuse to use it. +- **Timeouts**: It's important to adjust timeout values based on your network conditions to avoid connection issues. + +This configuration allows flexible SFTP authentication, whether you're using passwords, keys, or agents, and ensures secure communication between your local environment and the SFTP server. + ## Write disposition The filesystem destination handles the write dispositions as follows: - `append` - files belonging to such tables are added to the dataset folder diff --git a/poetry.lock b/poetry.lock index e1afddfd5f..8f2ff58094 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "about-time" @@ -923,6 +923,46 @@ test = ["beautifulsoup4 (>=4.8.0)", "coverage (>=4.5.4)", "fixtures (>=3.0.0)", toml = ["tomli (>=1.1.0)"] yaml = ["PyYAML"] +[[package]] +name = "bcrypt" +version = "4.2.0" +description = "Modern password hashing for your software and your servers" +optional = true +python-versions = ">=3.7" +files = [ + {file = "bcrypt-4.2.0-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:096a15d26ed6ce37a14c1ac1e48119660f21b24cba457f160a4b830f3fe6b5cb"}, + {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c02d944ca89d9b1922ceb8a46460dd17df1ba37ab66feac4870f6862a1533c00"}, + {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d84cf6d877918620b687b8fd1bf7781d11e8a0998f576c7aa939776b512b98d"}, + {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:1bb429fedbe0249465cdd85a58e8376f31bb315e484f16e68ca4c786dcc04291"}, + {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:655ea221910bcac76ea08aaa76df427ef8625f92e55a8ee44fbf7753dbabb328"}, + {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:1ee38e858bf5d0287c39b7a1fc59eec64bbf880c7d504d3a06a96c16e14058e7"}, + {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:0da52759f7f30e83f1e30a888d9163a81353ef224d82dc58eb5bb52efcabc399"}, + {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3698393a1b1f1fd5714524193849d0c6d524d33523acca37cd28f02899285060"}, + {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:762a2c5fb35f89606a9fde5e51392dad0cd1ab7ae64149a8b935fe8d79dd5ed7"}, + {file = "bcrypt-4.2.0-cp37-abi3-win32.whl", hash = "sha256:5a1e8aa9b28ae28020a3ac4b053117fb51c57a010b9f969603ed885f23841458"}, + {file = "bcrypt-4.2.0-cp37-abi3-win_amd64.whl", hash = "sha256:8f6ede91359e5df88d1f5c1ef47428a4420136f3ce97763e31b86dd8280fbdf5"}, + {file = "bcrypt-4.2.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:c52aac18ea1f4a4f65963ea4f9530c306b56ccd0c6f8c8da0c06976e34a6e841"}, + {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3bbbfb2734f0e4f37c5136130405332640a1e46e6b23e000eeff2ba8d005da68"}, + {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3413bd60460f76097ee2e0a493ccebe4a7601918219c02f503984f0a7ee0aebe"}, + {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8d7bb9c42801035e61c109c345a28ed7e84426ae4865511eb82e913df18f58c2"}, + {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3d3a6d28cb2305b43feac298774b997e372e56c7c7afd90a12b3dc49b189151c"}, + {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9c1c4ad86351339c5f320ca372dfba6cb6beb25e8efc659bedd918d921956bae"}, + {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:27fe0f57bb5573104b5a6de5e4153c60814c711b29364c10a75a54bb6d7ff48d"}, + {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8ac68872c82f1add6a20bd489870c71b00ebacd2e9134a8aa3f98a0052ab4b0e"}, + {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cb2a8ec2bc07d3553ccebf0746bbf3d19426d1c6d1adbd4fa48925f66af7b9e8"}, + {file = "bcrypt-4.2.0-cp39-abi3-win32.whl", hash = "sha256:77800b7147c9dc905db1cba26abe31e504d8247ac73580b4aa179f98e6608f34"}, + {file = "bcrypt-4.2.0-cp39-abi3-win_amd64.whl", hash = "sha256:61ed14326ee023917ecd093ee6ef422a72f3aec6f07e21ea5f10622b735538a9"}, + {file = "bcrypt-4.2.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:39e1d30c7233cfc54f5c3f2c825156fe044efdd3e0b9d309512cc514a263ec2a"}, + {file = "bcrypt-4.2.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f4f4acf526fcd1c34e7ce851147deedd4e26e6402369304220250598b26448db"}, + {file = "bcrypt-4.2.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:1ff39b78a52cf03fdf902635e4c81e544714861ba3f0efc56558979dd4f09170"}, + {file = "bcrypt-4.2.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:373db9abe198e8e2c70d12b479464e0d5092cc122b20ec504097b5f2297ed184"}, + {file = "bcrypt-4.2.0.tar.gz", hash = "sha256:cf69eaf5185fd58f268f805b505ce31f9b9fc2d64b376642164e9244540c1221"}, +] + +[package.extras] +tests = ["pytest (>=3.2.1,!=3.3.0)"] +typecheck = ["mypy"] + [[package]] name = "beautifulsoup4" version = "4.12.2" @@ -3749,106 +3789,6 @@ files = [ {file = "google_re2-1.1-4-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f4d4f0823e8b2f6952a145295b1ff25245ce9bb136aff6fe86452e507d4c1dd"}, {file = "google_re2-1.1-4-cp39-cp39-win32.whl", hash = "sha256:1afae56b2a07bb48cfcfefaa15ed85bae26a68f5dc7f9e128e6e6ea36914e847"}, {file = "google_re2-1.1-4-cp39-cp39-win_amd64.whl", hash = "sha256:aa7d6d05911ab9c8adbf3c225a7a120ab50fd2784ac48f2f0d140c0b7afc2b55"}, - {file = "google_re2-1.1-5-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:222fc2ee0e40522de0b21ad3bc90ab8983be3bf3cec3d349c80d76c8bb1a4beb"}, - {file = "google_re2-1.1-5-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:d4763b0b9195b72132a4e7de8e5a9bf1f05542f442a9115aa27cfc2a8004f581"}, - {file = "google_re2-1.1-5-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:209649da10c9d4a93d8a4d100ecbf9cc3b0252169426bec3e8b4ad7e57d600cf"}, - {file = "google_re2-1.1-5-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:68813aa333c1604a2df4a495b2a6ed065d7c8aebf26cc7e7abb5a6835d08353c"}, - {file = "google_re2-1.1-5-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:370a23ec775ad14e9d1e71474d56f381224dcf3e72b15d8ca7b4ad7dd9cd5853"}, - {file = "google_re2-1.1-5-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:14664a66a3ddf6bc9e56f401bf029db2d169982c53eff3f5876399104df0e9a6"}, - {file = "google_re2-1.1-5-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ea3722cc4932cbcebd553b69dce1b4a73572823cff4e6a244f1c855da21d511"}, - {file = "google_re2-1.1-5-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e14bb264c40fd7c627ef5678e295370cd6ba95ca71d835798b6e37502fc4c690"}, - {file = "google_re2-1.1-5-cp310-cp310-win32.whl", hash = "sha256:39512cd0151ea4b3969c992579c79b423018b464624ae955be685fc07d94556c"}, - {file = "google_re2-1.1-5-cp310-cp310-win_amd64.whl", hash = "sha256:ac66537aa3bc5504320d922b73156909e3c2b6da19739c866502f7827b3f9fdf"}, - {file = "google_re2-1.1-5-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5b5ea68d54890c9edb1b930dcb2658819354e5d3f2201f811798bbc0a142c2b4"}, - {file = "google_re2-1.1-5-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:33443511b6b83c35242370908efe2e8e1e7cae749c766b2b247bf30e8616066c"}, - {file = "google_re2-1.1-5-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:413d77bdd5ba0bfcada428b4c146e87707452ec50a4091ec8e8ba1413d7e0619"}, - {file = "google_re2-1.1-5-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:5171686e43304996a34baa2abcee6f28b169806d0e583c16d55e5656b092a414"}, - {file = "google_re2-1.1-5-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3b284db130283771558e31a02d8eb8fb756156ab98ce80035ae2e9e3a5f307c4"}, - {file = "google_re2-1.1-5-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:296e6aed0b169648dc4b870ff47bd34c702a32600adb9926154569ef51033f47"}, - {file = "google_re2-1.1-5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:38d50e68ead374160b1e656bbb5d101f0b95fb4cc57f4a5c12100155001480c5"}, - {file = "google_re2-1.1-5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2a0416a35921e5041758948bcb882456916f22845f66a93bc25070ef7262b72a"}, - {file = "google_re2-1.1-5-cp311-cp311-win32.whl", hash = "sha256:a1d59568bbb5de5dd56dd6cdc79907db26cce63eb4429260300c65f43469e3e7"}, - {file = "google_re2-1.1-5-cp311-cp311-win_amd64.whl", hash = "sha256:72f5a2f179648b8358737b2b493549370debd7d389884a54d331619b285514e3"}, - {file = "google_re2-1.1-5-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:cbc72c45937b1dc5acac3560eb1720007dccca7c9879138ff874c7f6baf96005"}, - {file = "google_re2-1.1-5-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:5fadd1417fbef7235fa9453dba4eb102e6e7d94b1e4c99d5fa3dd4e288d0d2ae"}, - {file = "google_re2-1.1-5-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:040f85c63cc02696485b59b187a5ef044abe2f99b92b4fb399de40b7d2904ccc"}, - {file = "google_re2-1.1-5-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:64e3b975ee6d9bbb2420494e41f929c1a0de4bcc16d86619ab7a87f6ea80d6bd"}, - {file = "google_re2-1.1-5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:8ee370413e00f4d828eaed0e83b8af84d7a72e8ee4f4bd5d3078bc741dfc430a"}, - {file = "google_re2-1.1-5-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:5b89383001079323f693ba592d7aad789d7a02e75adb5d3368d92b300f5963fd"}, - {file = "google_re2-1.1-5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63cb4fdfbbda16ae31b41a6388ea621510db82feb8217a74bf36552ecfcd50ad"}, - {file = "google_re2-1.1-5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ebedd84ae8be10b7a71a16162376fd67a2386fe6361ef88c622dcf7fd679daf"}, - {file = "google_re2-1.1-5-cp312-cp312-win32.whl", hash = "sha256:c8e22d1692bc2c81173330c721aff53e47ffd3c4403ff0cd9d91adfd255dd150"}, - {file = "google_re2-1.1-5-cp312-cp312-win_amd64.whl", hash = "sha256:5197a6af438bb8c4abda0bbe9c4fbd6c27c159855b211098b29d51b73e4cbcf6"}, - {file = "google_re2-1.1-5-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:b6727e0b98417e114b92688ad2aa256102ece51f29b743db3d831df53faf1ce3"}, - {file = "google_re2-1.1-5-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:711e2b6417eb579c61a4951029d844f6b95b9b373b213232efd413659889a363"}, - {file = "google_re2-1.1-5-cp38-cp38-macosx_13_0_arm64.whl", hash = "sha256:71ae8b3df22c5c154c8af0f0e99d234a450ef1644393bc2d7f53fc8c0a1e111c"}, - {file = "google_re2-1.1-5-cp38-cp38-macosx_13_0_x86_64.whl", hash = "sha256:94a04e214bc521a3807c217d50cf099bbdd0c0a80d2d996c0741dbb995b5f49f"}, - {file = "google_re2-1.1-5-cp38-cp38-macosx_14_0_arm64.whl", hash = "sha256:a770f75358508a9110c81a1257721f70c15d9bb592a2fb5c25ecbd13566e52a5"}, - {file = "google_re2-1.1-5-cp38-cp38-macosx_14_0_x86_64.whl", hash = "sha256:07c9133357f7e0b17c6694d5dcb82e0371f695d7c25faef2ff8117ef375343ff"}, - {file = "google_re2-1.1-5-cp38-cp38-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:204ca6b1cf2021548f4a9c29ac015e0a4ab0a7b6582bf2183d838132b60c8fda"}, - {file = "google_re2-1.1-5-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0b95857c2c654f419ca684ec38c9c3325c24e6ba7d11910a5110775a557bb18"}, - {file = "google_re2-1.1-5-cp38-cp38-win32.whl", hash = "sha256:347ac770e091a0364e822220f8d26ab53e6fdcdeaec635052000845c5a3fb869"}, - {file = "google_re2-1.1-5-cp38-cp38-win_amd64.whl", hash = "sha256:ec32bb6de7ffb112a07d210cf9f797b7600645c2d5910703fa07f456dd2150e0"}, - {file = "google_re2-1.1-5-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:eb5adf89060f81c5ff26c28e261e6b4997530a923a6093c9726b8dec02a9a326"}, - {file = "google_re2-1.1-5-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:a22630c9dd9ceb41ca4316bccba2643a8b1d5c198f21c00ed5b50a94313aaf10"}, - {file = "google_re2-1.1-5-cp39-cp39-macosx_13_0_arm64.whl", hash = "sha256:544dc17fcc2d43ec05f317366375796351dec44058e1164e03c3f7d050284d58"}, - {file = "google_re2-1.1-5-cp39-cp39-macosx_13_0_x86_64.whl", hash = "sha256:19710af5ea88751c7768575b23765ce0dfef7324d2539de576f75cdc319d6654"}, - {file = "google_re2-1.1-5-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:f82995a205e08ad896f4bd5ce4847c834fab877e1772a44e5f262a647d8a1dec"}, - {file = "google_re2-1.1-5-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:63533c4d58da9dc4bc040250f1f52b089911699f0368e0e6e15f996387a984ed"}, - {file = "google_re2-1.1-5-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79e00fcf0cb04ea35a22b9014712d448725ce4ddc9f08cc818322566176ca4b0"}, - {file = "google_re2-1.1-5-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc41afcefee2da6c4ed883a93d7f527c4b960cd1d26bbb0020a7b8c2d341a60a"}, - {file = "google_re2-1.1-5-cp39-cp39-win32.whl", hash = "sha256:486730b5e1f1c31b0abc6d80abe174ce4f1188fe17d1b50698f2bf79dc6e44be"}, - {file = "google_re2-1.1-5-cp39-cp39-win_amd64.whl", hash = "sha256:4de637ca328f1d23209e80967d1b987d6b352cd01b3a52a84b4d742c69c3da6c"}, - {file = "google_re2-1.1-6-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:621e9c199d1ff0fdb2a068ad450111a84b3bf14f96dfe5a8a7a0deae5f3f4cce"}, - {file = "google_re2-1.1-6-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:220acd31e7dde95373f97c3d1f3b3bd2532b38936af28b1917ee265d25bebbf4"}, - {file = "google_re2-1.1-6-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:db34e1098d164f76251a6ece30e8f0ddfd65bb658619f48613ce71acb3f9cbdb"}, - {file = "google_re2-1.1-6-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:5152bac41d8073977582f06257219541d0fc46ad99b0bbf30e8f60198a43b08c"}, - {file = "google_re2-1.1-6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:6191294799e373ee1735af91f55abd23b786bdfd270768a690d9d55af9ea1b0d"}, - {file = "google_re2-1.1-6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:070cbafbb4fecbb02e98feb28a1eb292fb880f434d531f38cc33ee314b521f1f"}, - {file = "google_re2-1.1-6-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8437d078b405a59a576cbed544490fe041140f64411f2d91012e8ec05ab8bf86"}, - {file = "google_re2-1.1-6-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f00f9a9af8896040e37896d9b9fc409ad4979f1ddd85bb188694a7d95ddd1164"}, - {file = "google_re2-1.1-6-cp310-cp310-win32.whl", hash = "sha256:df26345f229a898b4fd3cafd5f82259869388cee6268fc35af16a8e2293dd4e5"}, - {file = "google_re2-1.1-6-cp310-cp310-win_amd64.whl", hash = "sha256:3665d08262c57c9b28a5bdeb88632ad792c4e5f417e5645901695ab2624f5059"}, - {file = "google_re2-1.1-6-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b26b869d8aa1d8fe67c42836bf3416bb72f444528ee2431cfb59c0d3e02c6ce3"}, - {file = "google_re2-1.1-6-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:41fd4486c57dea4f222a6bb7f1ff79accf76676a73bdb8da0fcbd5ba73f8da71"}, - {file = "google_re2-1.1-6-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:0ee378e2e74e25960070c338c28192377c4dd41e7f4608f2688064bd2badc41e"}, - {file = "google_re2-1.1-6-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:a00cdbf662693367b36d075b29feb649fd7ee1b617cf84f85f2deebeda25fc64"}, - {file = "google_re2-1.1-6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4c09455014217a41499432b8c8f792f25f3df0ea2982203c3a8c8ca0e7895e69"}, - {file = "google_re2-1.1-6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:6501717909185327935c7945e23bb5aa8fc7b6f237b45fe3647fa36148662158"}, - {file = "google_re2-1.1-6-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3510b04790355f199e7861c29234081900e1e1cbf2d1484da48aa0ba6d7356ab"}, - {file = "google_re2-1.1-6-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8c0e64c187ca406764f9e9ad6e750d62e69ed8f75bf2e865d0bfbc03b642361c"}, - {file = "google_re2-1.1-6-cp311-cp311-win32.whl", hash = "sha256:2a199132350542b0de0f31acbb3ca87c3a90895d1d6e5235f7792bb0af02e523"}, - {file = "google_re2-1.1-6-cp311-cp311-win_amd64.whl", hash = "sha256:83bdac8ceaece8a6db082ea3a8ba6a99a2a1ee7e9f01a9d6d50f79c6f251a01d"}, - {file = "google_re2-1.1-6-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:81985ff894cd45ab5a73025922ac28c0707759db8171dd2f2cc7a0e856b6b5ad"}, - {file = "google_re2-1.1-6-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:5635af26065e6b45456ccbea08674ae2ab62494008d9202df628df3b267bc095"}, - {file = "google_re2-1.1-6-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:813b6f04de79f4a8fdfe05e2cb33e0ccb40fe75d30ba441d519168f9d958bd54"}, - {file = "google_re2-1.1-6-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:5ec2f5332ad4fd232c3f2d6748c2c7845ccb66156a87df73abcc07f895d62ead"}, - {file = "google_re2-1.1-6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:5a687b3b32a6cbb731647393b7c4e3fde244aa557f647df124ff83fb9b93e170"}, - {file = "google_re2-1.1-6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:39a62f9b3db5d3021a09a47f5b91708b64a0580193e5352751eb0c689e4ad3d7"}, - {file = "google_re2-1.1-6-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ca0f0b45d4a1709cbf5d21f355e5809ac238f1ee594625a1e5ffa9ff7a09eb2b"}, - {file = "google_re2-1.1-6-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a64b3796a7a616c7861247bd061c9a836b5caf0d5963e5ea8022125601cf7b09"}, - {file = "google_re2-1.1-6-cp312-cp312-win32.whl", hash = "sha256:32783b9cb88469ba4cd9472d459fe4865280a6b1acdad4480a7b5081144c4eb7"}, - {file = "google_re2-1.1-6-cp312-cp312-win_amd64.whl", hash = "sha256:259ff3fd2d39035b9cbcbf375995f83fa5d9e6a0c5b94406ff1cc168ed41d6c6"}, - {file = "google_re2-1.1-6-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:e4711bcffe190acd29104d8ecfea0c0e42b754837de3fb8aad96e6cc3c613cdc"}, - {file = "google_re2-1.1-6-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:4d081cce43f39c2e813fe5990e1e378cbdb579d3f66ded5bade96130269ffd75"}, - {file = "google_re2-1.1-6-cp38-cp38-macosx_13_0_arm64.whl", hash = "sha256:4f123b54d48450d2d6b14d8fad38e930fb65b5b84f1b022c10f2913bd956f5b5"}, - {file = "google_re2-1.1-6-cp38-cp38-macosx_13_0_x86_64.whl", hash = "sha256:e1928b304a2b591a28eb3175f9db7f17c40c12cf2d4ec2a85fdf1cc9c073ff91"}, - {file = "google_re2-1.1-6-cp38-cp38-macosx_14_0_arm64.whl", hash = "sha256:3a69f76146166aec1173003c1f547931bdf288c6b135fda0020468492ac4149f"}, - {file = "google_re2-1.1-6-cp38-cp38-macosx_14_0_x86_64.whl", hash = "sha256:fc08c388f4ebbbca345e84a0c56362180d33d11cbe9ccfae663e4db88e13751e"}, - {file = "google_re2-1.1-6-cp38-cp38-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b057adf38ce4e616486922f2f47fc7d19c827ba0a7f69d540a3664eba2269325"}, - {file = "google_re2-1.1-6-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4138c0b933ab099e96f5d8defce4486f7dfd480ecaf7f221f2409f28022ccbc5"}, - {file = "google_re2-1.1-6-cp38-cp38-win32.whl", hash = "sha256:9693e45b37b504634b1abbf1ee979471ac6a70a0035954592af616306ab05dd6"}, - {file = "google_re2-1.1-6-cp38-cp38-win_amd64.whl", hash = "sha256:5674d437baba0ea287a5a7f8f81f24265d6ae8f8c09384e2ef7b6f84b40a7826"}, - {file = "google_re2-1.1-6-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:7783137cb2e04f458a530c6d0ee9ef114815c1d48b9102f023998c371a3b060e"}, - {file = "google_re2-1.1-6-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:a49b7153935e7a303675f4deb5f5d02ab1305adefc436071348706d147c889e0"}, - {file = "google_re2-1.1-6-cp39-cp39-macosx_13_0_arm64.whl", hash = "sha256:a96a8bb309182090704593c60bdb369a2756b38fe358bbf0d40ddeb99c71769f"}, - {file = "google_re2-1.1-6-cp39-cp39-macosx_13_0_x86_64.whl", hash = "sha256:dff3d4be9f27ef8ec3705eed54f19ef4ab096f5876c15fe011628c69ba3b561c"}, - {file = "google_re2-1.1-6-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:40f818b0b39e26811fa677978112a8108269977fdab2ba0453ac4363c35d9e66"}, - {file = "google_re2-1.1-6-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:8a7e53538cdb40ef4296017acfbb05cab0c19998be7552db1cfb85ba40b171b9"}, - {file = "google_re2-1.1-6-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ee18e7569fb714e5bb8c42809bf8160738637a5e71ed5a4797757a1fb4dc4de"}, - {file = "google_re2-1.1-6-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1cda4f6d1a7d5b43ea92bc395f23853fba0caf8b1e1efa6e8c48685f912fcb89"}, - {file = "google_re2-1.1-6-cp39-cp39-win32.whl", hash = "sha256:6a9cdbdc36a2bf24f897be6a6c85125876dc26fea9eb4247234aec0decbdccfd"}, - {file = "google_re2-1.1-6-cp39-cp39-win_amd64.whl", hash = "sha256:73f646cecfad7cc5b4330b4192c25f2e29730a3b8408e089ffd2078094208196"}, ] [[package]] @@ -6279,6 +6219,27 @@ sql-other = ["SQLAlchemy (>=1.4.36)"] test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] xml = ["lxml (>=4.8.0)"] +[[package]] +name = "paramiko" +version = "3.4.1" +description = "SSH2 protocol library" +optional = true +python-versions = ">=3.6" +files = [ + {file = "paramiko-3.4.1-py3-none-any.whl", hash = "sha256:8e49fd2f82f84acf7ffd57c64311aa2b30e575370dc23bdb375b10262f7eac32"}, + {file = "paramiko-3.4.1.tar.gz", hash = "sha256:8b15302870af7f6652f2e038975c1d2973f06046cb5d7d65355668b3ecbece0c"}, +] + +[package.dependencies] +bcrypt = ">=3.2" +cryptography = ">=3.3" +pynacl = ">=1.5" + +[package.extras] +all = ["gssapi (>=1.4.1)", "invoke (>=2.0)", "pyasn1 (>=0.1.7)", "pywin32 (>=2.1.8)"] +gssapi = ["gssapi (>=1.4.1)", "pyasn1 (>=0.1.7)", "pywin32 (>=2.1.8)"] +invoke = ["invoke (>=2.0)"] + [[package]] name = "parsedatetime" version = "2.4" @@ -7170,6 +7131,32 @@ files = [ ed25519 = ["PyNaCl (>=1.4.0)"] rsa = ["cryptography"] +[[package]] +name = "pynacl" +version = "1.5.0" +description = "Python binding to the Networking and Cryptography (NaCl) library" +optional = true +python-versions = ">=3.6" +files = [ + {file = "PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858"}, + {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b"}, + {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff"}, + {file = "PyNaCl-1.5.0-cp36-abi3-win32.whl", hash = "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543"}, + {file = "PyNaCl-1.5.0-cp36-abi3-win_amd64.whl", hash = "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93"}, + {file = "PyNaCl-1.5.0.tar.gz", hash = "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba"}, +] + +[package.dependencies] +cffi = ">=1.4.1" + +[package.extras] +docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] +tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] + [[package]] name = "pyodbc" version = "4.0.39" @@ -9762,6 +9749,7 @@ postgres = ["psycopg2-binary", "psycopg2cffi"] qdrant = ["qdrant-client"] redshift = ["psycopg2-binary", "psycopg2cffi"] s3 = ["botocore", "s3fs"] +sftp = ["paramiko"] snowflake = ["snowflake-connector-python"] sql-database = ["sqlalchemy"] sqlalchemy = ["alembic", "sqlalchemy"] @@ -9771,4 +9759,4 @@ weaviate = ["weaviate-client"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.13" -content-hash = "e6409812458e0aae06a9b5f2c816b19d73645c8598a1789b8d5b39a45e974a9f" +content-hash = "cf2b7cd45b7127328b25128320607b25a2c3b869f2ee6f79412fa12dc56441eb" diff --git a/pyproject.toml b/pyproject.toml index df0bc11782..3f973b6b65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ tantivy = { version = ">= 0.22.0", optional = true } deltalake = { version = ">=0.19.0", optional = true } sqlalchemy = { version = ">=1.4", optional = true } alembic = {version = "^1.13.2", optional = true} +paramiko = {version = ">=3.3.0", optional = true} [tool.poetry.extras] gcp = ["grpcio", "google-cloud-bigquery", "db-dtypes", "gcsfs"] @@ -98,6 +99,7 @@ filesystem = ["s3fs", "botocore"] s3 = ["s3fs", "botocore"] gs = ["gcsfs"] az = ["adlfs"] +sftp = ["paramiko"] snowflake = ["snowflake-connector-python"] motherduck = ["duckdb", "pyarrow"] cli = ["pipdeptree", "cron-descriptor"] diff --git a/pytest.ini b/pytest.ini index 1d4e0df6dc..4f033f672c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -11,4 +11,5 @@ filterwarnings= ignore::DeprecationWarning markers = essential: marks all essential tests no_load: marks tests that do not load anything - needspyarrow17: marks tests that need pyarrow>=17.0.0 (deselected by default) \ No newline at end of file + needspyarrow17: marks tests that need pyarrow>=17.0.0 (deselected by default) + \ No newline at end of file diff --git a/tests/.dlt/config.toml b/tests/.dlt/config.toml index 292175569b..62bfbc7680 100644 --- a/tests/.dlt/config.toml +++ b/tests/.dlt/config.toml @@ -10,4 +10,5 @@ bucket_url_abfss="abfss://dlt-ci-test-bucket@dltdata.dfs.core.windows.net" bucket_url_r2="s3://dlt-ci-test-bucket" # use "/" as root path bucket_url_gdrive="gdrive://15eC3e5MNew2XAIefWNlG8VlEa0ISnnaG" +bucket_url_sftp="sftp://localhost/data" memory="memory:///m" \ No newline at end of file diff --git a/tests/load/filesystem/test_filesystem_client.py b/tests/load/filesystem/test_filesystem_client.py index f16e75c7e6..fdd68d4683 100644 --- a/tests/load/filesystem/test_filesystem_client.py +++ b/tests/load/filesystem/test_filesystem_client.py @@ -194,9 +194,10 @@ def test_replace_write_disposition(layout: str, default_buckets_env: str) -> Non # First file from load1 remains, second file is replaced by load2 # assert that only these two files are in the destination folder + is_sftp = urlparse(default_buckets_env).scheme == "sftp" paths = [] for basedir, _dirs, files in client.fs_client.walk( - client.dataset_path, detail=False, refresh=True + client.dataset_path, detail=False, **({"refresh": True} if not is_sftp else {}) ): # remove internal paths if "_dlt" in basedir: @@ -257,9 +258,10 @@ def test_append_write_disposition(layout: str, default_buckets_env: str) -> None ] expected_files = sorted([Path(posixpath.join(root_path, fn)) for fn in expected_files]) # type: ignore[misc] + is_sftp = urlparse(default_buckets_env).scheme == "sftp" paths = [] for basedir, _dirs, files in client.fs_client.walk( - client.dataset_path, detail=False, refresh=True + client.dataset_path, detail=False, **({"refresh": True} if not is_sftp else {}) ): # remove internal paths if "_dlt" in basedir: diff --git a/tests/load/filesystem_sftp/__init__.py b/tests/load/filesystem_sftp/__init__.py new file mode 100644 index 0000000000..0d23f8002b --- /dev/null +++ b/tests/load/filesystem_sftp/__init__.py @@ -0,0 +1,3 @@ +from tests.utils import skip_if_not_active + +skip_if_not_active("filesystem") diff --git a/tests/load/filesystem_sftp/bootstrap/Dockerfile b/tests/load/filesystem_sftp/bootstrap/Dockerfile new file mode 100644 index 0000000000..a6c75d8c88 --- /dev/null +++ b/tests/load/filesystem_sftp/bootstrap/Dockerfile @@ -0,0 +1,117 @@ +FROM ubuntu:noble + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y openssh-server && \ + rm -rf /var/lib/apt/lists/* + +# Certificate Authority (CA): public key +COPY ca_rsa.pub /etc/ssh/ca_rsa.pub +RUN chmod 600 /etc/ssh/ca_rsa.pub + +RUN mkdir -p /etc/ssh/auth_principals && \ + echo "billy" >> /etc/ssh/auth_principals/billy + +RUN mkdir -p /run/sshd && \ + echo "SyslogFacility AUTH" >> /etc/ssh/sshd_config && \ + echo "LogLevel INFO" >> /etc/ssh/sshd_config && \ + echo "TrustedUserCAKeys /etc/ssh/ca_rsa.pub" >> /etc/ssh/sshd_config && \ + echo "AuthorizedPrincipalsFile /etc/ssh/auth_principals/billy" >> /etc/ssh/sshd_config + +# USER foo: set up user for SFTP with no shell login +RUN useradd -m -d /home/foo -s /usr/sbin/nologin foo && \ + mkdir -p /home/foo/.ssh && \ + chown foo:foo /home/foo/.ssh && \ + chmod 700 /home/foo/.ssh + +# USER foo: set password for the user foo +RUN echo 'foo:pass' | chpasswd + +# USER foo: copy the public key and set permissions +COPY foo_rsa.pub /home/foo/.ssh/authorized_keys +RUN chmod 600 /home/foo/.ssh/authorized_keys && \ + chown foo:foo /home/foo/.ssh/authorized_keys + +# USER foo: create a directory for SFTP that the user will have access to +RUN mkdir -p /home/foo/sftp/data && \ + chown root:root /home/foo /home/foo/sftp && \ + chmod 755 /home/foo /home/foo/sftp && \ + chown foo:foo /home/foo/sftp/data && \ + chmod 755 /home/foo/sftp/data + + +# USER foo: configure SSH for SFTP: allow password and pubkey authentication +RUN echo "Match User foo" >> /etc/ssh/sshd_config && \ + echo " ChrootDirectory /home/foo/sftp" >> /etc/ssh/sshd_config && \ + echo " ForceCommand internal-sftp" >> /etc/ssh/sshd_config && \ + echo " PasswordAuthentication yes" >> /etc/ssh/sshd_config && \ + echo " PubkeyAuthentication yes" >> /etc/ssh/sshd_config && \ + echo " PermitTunnel no" >> /etc/ssh/sshd_config && \ + echo " AllowAgentForwarding no" >> /etc/ssh/sshd_config && \ + echo " AllowTcpForwarding no" >> /etc/ssh/sshd_config && \ + echo " X11Forwarding no" >> /etc/ssh/sshd_config + +# USER bobby: set up user for SFTP with no shell login +RUN useradd -m -d /home/bobby -s /usr/sbin/nologin bobby && \ + mkdir -p /home/bobby/.ssh && \ + chown bobby:bobby /home/bobby/.ssh && \ + chmod 700 /home/bobby/.ssh + +# USER bobby: copy the public key and set permissions +COPY bobby_rsa.pub /home/bobby/.ssh/authorized_keys +RUN chmod 600 /home/bobby/.ssh/authorized_keys && \ + chown bobby:bobby /home/bobby/.ssh/authorized_keys + +# USER bobby: create a directory for SFTP that the user will have access to +RUN mkdir -p /home/bobby/sftp/data && \ + chown root:root /home/bobby /home/bobby/sftp && \ + chmod 755 /home/bobby /home/bobby/sftp && \ + chown bobby:bobby /home/bobby/sftp/data && \ + chmod 755 /home/bobby/sftp/data + +# USER bobby: configure SSH for SFTP: allow password and pubkey authentication +RUN echo "Match User bobby" >> /etc/ssh/sshd_config && \ + echo " ChrootDirectory /home/bobby/sftp" >> /etc/ssh/sshd_config && \ + echo " ForceCommand internal-sftp" >> /etc/ssh/sshd_config && \ + echo " PasswordAuthentication no" >> /etc/ssh/sshd_config && \ + echo " PubkeyAuthentication yes" >> /etc/ssh/sshd_config && \ + echo " PermitTunnel no" >> /etc/ssh/sshd_config && \ + echo " AllowAgentForwarding no" >> /etc/ssh/sshd_config && \ + echo " AllowTcpForwarding no" >> /etc/ssh/sshd_config && \ + echo " X11Forwarding no" >> /etc/ssh/sshd_config + +# USER billy: set up user for SFTP with no shell login +RUN useradd -m -d /home/billy -s /usr/sbin/nologin billy && \ + mkdir -p /home/billy/.ssh && \ + chown billy:billy /home/billy/.ssh && \ + chmod 700 /home/billy/.ssh + +# USER billy: create a directory for SFTP that the user will have access to +RUN mkdir -p /home/billy/sftp/data && \ + chown root:root /home/billy /home/billy/sftp && \ + chmod 755 /home/billy /home/billy/sftp && \ + chown billy:billy /home/billy/sftp/data && \ + chmod 755 /home/billy/sftp/data + +# USER billy: certificated signed with CA key +COPY billy_rsa-cert.pub /home/billy/.ssh/billy_rsa-cert.pub + +RUN chown billy:billy /home/billy/.ssh/billy_rsa-cert.pub && \ + chmod 600 /home/billy/.ssh/billy_rsa-cert.pub + +# USER billy: configure SSH for SFTP with certificate authentication +RUN echo "Match User billy" >> /etc/ssh/sshd_config && \ + echo " ChrootDirectory /home/billy/sftp" >> /etc/ssh/sshd_config && \ + echo " ForceCommand internal-sftp" >> /etc/ssh/sshd_config && \ + echo " PasswordAuthentication no" >> /etc/ssh/sshd_config && \ + echo " PubkeyAuthentication yes" >> /etc/ssh/sshd_config && \ + echo " PermitTunnel no" >> /etc/ssh/sshd_config && \ + echo " AllowAgentForwarding no" >> /etc/ssh/sshd_config && \ + echo " AllowTcpForwarding no" >> /etc/ssh/sshd_config && \ + echo " X11Forwarding no" >> /etc/ssh/sshd_config + +EXPOSE 22 + +# run sshd on container start +CMD ["/usr/sbin/sshd", "-D", "-e"] diff --git a/tests/load/filesystem_sftp/bootstrap/SETUP.md b/tests/load/filesystem_sftp/bootstrap/SETUP.md new file mode 100644 index 0000000000..0620bbb075 --- /dev/null +++ b/tests/load/filesystem_sftp/bootstrap/SETUP.md @@ -0,0 +1,49 @@ +## Users: Authentication + +This guide covers the setup of different authentication methods for SSH users, including public/private key pairs, passphrase protection, and certificate-based authentication. + +### User foo: Public/Private Key Pair Without Passphrase + +Generate a key pair for `foo` without a passphrase: +```bash +# Generate the key pair +ssh-keygen -t rsa -b 4096 -C "foo@example.com" -f foo_rsa + +# Secure the private key +chmod 600 foo_rsa +``` + +### User bobby: Public/Private Key Pair With Passphrase + +Generate a key pair for `bobby` with a passphrase (passphrase=passphrase123): +```bash +# Generate the key pair with a passphrase +ssh-keygen -t rsa -b 4096 -C "bobby@example.com" -f bobby_rsa + +# Secure the private key +chmod 600 bobby_rsa +``` + +### Certificate Authority (CA) Setup + +Generate the Certificate Authority (CA) key pair: +```bash +# Generate a self-signed CA key pair +ssh-keygen -t rsa -b 4096 -f ca_rsa -N "" +``` + +### User billy: Public/Private Key Pair with CA-Signed Certificate + +Generate and sign a key pair for `billy` using the CA: +```bash +# Generate the user key pair for billy +ssh-keygen -t rsa -b 4096 -C "billy@example.com" -f billy_rsa + +# Sign billy's public key with the CA +ssh-keygen -s ca_rsa -I billy-cert -n billy billy_rsa.pub +``` + +### Important Files + +- **ca_rsa.pub**: The CA public key. This key is used by the server to verify certificates. +- **billy_rsa-cert.pub**: Billy’s signed certificate. This certificate is used by Billy to authenticate with the server. diff --git a/tests/load/filesystem_sftp/bootstrap/billy_rsa b/tests/load/filesystem_sftp/bootstrap/billy_rsa new file mode 100644 index 0000000000..ceafa496e0 --- /dev/null +++ b/tests/load/filesystem_sftp/bootstrap/billy_rsa @@ -0,0 +1,49 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn +NhAAAAAwEAAQAAAgEAt6vSxFUsoMmftSc2sn7yd/GqN1QuD2DlCzdp+MNF2awKe4JXC+9v +OKDwYWk1DU1Z2r400hUFpsColXrO7kNJKdLNZw7tUcIwusXUWobh3zT0VCpompycJI1ylZ +HTX7cSwuSX43VRNmsj049sawTtwb+5yURsZuc8fLRX4x0EYjbqrP4TxEvumvXf2vjPQg+l +d4QCASTWNijN0aBRV+fnfCXlkkt+9PhyqUYFmLfd//JclEs5JEyOmsX68tUTx90XNKC4ef +0vYDnQsMTDujHOwjla9ZTVMvhXBg10ZUd+rmbWlwD9gCiIqL5YZg/hsJ9QEVYr+03hEq+q +nBAqRTwvFXaJHGUVNg7eCkE3w4RFSqU7FIwag9naHBQvbW0LLMAIpJqRPQ81J0LJtxXPsi +X+xt4Oah+jgWMAXtKkfzzZfu+BRDZ4F/TqRj7xgPrwsCgIFE6n3PpgnBRZb1IczQ0VFuGX +O2kvCpH9bJ5Rb8MEJyZ1z/GqiagBfDhUIXSzGLSxjGfNR6bmrrBL2J3PPviAgd1epjFhMj +nkeWysT9U17PoyiJQlVnwEAQo04pEWQLhrVBz0DSjH9noElhVURvvHClfkgYA1HrSCz+qH +Lu+1vgo9S/+14i2q0F/ILPGJob7dlZNIxZcLVnN2U72/1WS3z0DZ2xr7kMSqMQ3QiMzrez +UAAAdImHtW95h7VvcAAAAHc3NoLXJzYQAAAgEAt6vSxFUsoMmftSc2sn7yd/GqN1QuD2Dl +Czdp+MNF2awKe4JXC+9vOKDwYWk1DU1Z2r400hUFpsColXrO7kNJKdLNZw7tUcIwusXUWo +bh3zT0VCpompycJI1ylZHTX7cSwuSX43VRNmsj049sawTtwb+5yURsZuc8fLRX4x0EYjbq +rP4TxEvumvXf2vjPQg+ld4QCASTWNijN0aBRV+fnfCXlkkt+9PhyqUYFmLfd//JclEs5JE +yOmsX68tUTx90XNKC4ef0vYDnQsMTDujHOwjla9ZTVMvhXBg10ZUd+rmbWlwD9gCiIqL5Y +Zg/hsJ9QEVYr+03hEq+qnBAqRTwvFXaJHGUVNg7eCkE3w4RFSqU7FIwag9naHBQvbW0LLM +AIpJqRPQ81J0LJtxXPsiX+xt4Oah+jgWMAXtKkfzzZfu+BRDZ4F/TqRj7xgPrwsCgIFE6n +3PpgnBRZb1IczQ0VFuGXO2kvCpH9bJ5Rb8MEJyZ1z/GqiagBfDhUIXSzGLSxjGfNR6bmrr +BL2J3PPviAgd1epjFhMjnkeWysT9U17PoyiJQlVnwEAQo04pEWQLhrVBz0DSjH9noElhVU +RvvHClfkgYA1HrSCz+qHLu+1vgo9S/+14i2q0F/ILPGJob7dlZNIxZcLVnN2U72/1WS3z0 +DZ2xr7kMSqMQ3QiMzrezUAAAADAQABAAACAGJcgTYliw3cjEv+YRt8QqubXxXcXgJcWK0G +ExlmsgydRsvYBHfe1QhVoAX6yq+CAiHUBQOQwIMJ3/6VUyZkIFVmkBul8c/8GA6eDVghre +rUOBiu0XAjeLdWjMuu2a0DpOd8dh8Y39It1HTi9SPm8tD0tTElFIyTP2BlUD/PdV6HnGvi +cYGJTJHuJODmqE/vgySbLFuid5lvhrHBkm5qQhzJSsVq6uWT/AvM9QhRR3FfQNcl5ccbvK +8EYZ0UIVDChT/o5oJl5a45qBpSdaNMMzbucljcSMTQHAKgx/nJLP2HE1qw7BFrI6yCq2KV +FL96we1W4ZzmmydfrhV/zrNfRLLohPv6vbEho/cuMfZAwtdHunYJSV3e5bBNKbjTAIh7O2 +nIBDHLUFhRUZHJcWcs7n0gQgmD2cpIT71fyTo3+RV0hq1j6p8idoFxG3O+gs1a3tJ+3Koj +J1+2r2ocUkY2f2UJZtP2dE8c2ZK5ttd7rqv6s4d9sHn6n87hn4UKVf2O5Y7Mlje/v5aTn2 +9KjR0pkGe3WdMcBl8n7usq04TjBNChYjUk1l1dQ4Te66OQ6XDCqnLJxETPTgsa3iVyYjR0 +CNuleqW+fHGbd7RhmWS6mlTHKYo5dCcilBNxrFQUyJpPPGyDZdZRiYWk5/yqbSs30k20mH +gA+TJoiBiDRJYvySRtAAABABeBxEbk8oGH1cK1+4w4kpAg5zZ7cAFuibMOP3TnxQNIElBF +RGFGZJVGE+moYiHRWHBKtEbTKVqA+LUqStE2dj6wjxg7d6qdtUm/7Dw6ODSvWCI7YXTvys +YsWuktbmqii3VIy6ZOo6Hp0jdQOKnZni/Es9gfSpKA3f6vmc9sjiIFJBINhOOu7l7XiaCx +6tHuhypqAmnrQxtso7Pu2WEofNCwHc/c2QBpVTW32QOpfHc5MZ0gt9V4ozxhDa8sP6QQqA +h2KhOWc2tU0kIaespfoeFKPmNTbDVXT8uiJqHE6+RkEtlLFqLuL9o2U9mPErG1d5HUUkUj +HhtI3rWqeESPDEoAAAEBAN3U9dVp0G84XVtdPkFV8d5CxJZgM6Wwse1ZzGyy9GcO2vUhE8 +H+MistGl/18JA89UqkPEuWOVTM2ISj8mHpXBiEHOEQpm53sktO9BBBrqlfoUQ4/pgOuMzx +TvZozZ22XhyX+b4oVdwAcqzV9u/QwaPDZn5SKdVDNznbuFba0Oz5OcPT/O/0doTldMwiwI +PN4ptoCH+4b5gNKr6lcEtiFHwKJ89XUK0J3DIufcykjSQ0Ff/xeVR8C5yK44xTni26tpYS +VHJbJ5PiaLyGnIeBrA7atKtzayz1Vt+8h7RLkoFim2QT9782763CfFDyqzZndill089dwV +wg10ObtbNLaWMAAAEBANP2JqpTDzVV25BspY3o9tQK9wopr3DFfAQV2nG02e1ygwHgRcgA +hkEFYidJX3K4BbVaVoP19D0So1ERjXsmbDiTRgXRgZQteX+5xMDMDUMyCOZVY/gQPB7K3E +UeLHaHQiAqqwz9qhbtvfvjBkKwg6HgF+EfB97eFwBzkACYa+xQQrHTOpwM666vlEnKI6AI +8+KgOpzHs04cowIm0sUUVcn1eeOj8RTa7KAPp06hWUcQnCO9+Pb1hCS1GvXkGNUv0K4h1T +mogsfGdRKbeyG2izaPQde5pykB0G7INqzFZJ5Rgydc0khOMHFiv8grDKH43csa3IUMailM +8OBopUpl+IcAAAARYmlsbHlAZXhhbXBsZS5jb20BAg== +-----END OPENSSH PRIVATE KEY----- diff --git a/tests/load/filesystem_sftp/bootstrap/billy_rsa-cert.pub b/tests/load/filesystem_sftp/bootstrap/billy_rsa-cert.pub new file mode 100644 index 0000000000..147113c067 --- /dev/null +++ b/tests/load/filesystem_sftp/bootstrap/billy_rsa-cert.pub @@ -0,0 +1 @@ +ssh-rsa-cert-v01@openssh.com AAAAHHNzaC1yc2EtY2VydC12MDFAb3BlbnNzaC5jb20AAAAgBOH3dEBDItfrfBWyMDp3/eonWDzVrHs5NAVMTk6EcYEAAAADAQABAAACAQC3q9LEVSygyZ+1JzayfvJ38ao3VC4PYOULN2n4w0XZrAp7glcL7284oPBhaTUNTVnavjTSFQWmwKiVes7uQ0kp0s1nDu1RwjC6xdRahuHfNPRUKmianJwkjXKVkdNftxLC5JfjdVE2ayPTj2xrBO3Bv7nJRGxm5zx8tFfjHQRiNuqs/hPES+6a9d/a+M9CD6V3hAIBJNY2KM3RoFFX5+d8JeWSS370+HKpRgWYt93/8lyUSzkkTI6axfry1RPH3Rc0oLh5/S9gOdCwxMO6Mc7COVr1lNUy+FcGDXRlR36uZtaXAP2AKIiovlhmD+Gwn1ARViv7TeESr6qcECpFPC8VdokcZRU2Dt4KQTfDhEVKpTsUjBqD2docFC9tbQsswAikmpE9DzUnQsm3Fc+yJf7G3g5qH6OBYwBe0qR/PNl+74FENngX9OpGPvGA+vCwKAgUTqfc+mCcFFlvUhzNDRUW4Zc7aS8Kkf1snlFvwwQnJnXP8aqJqAF8OFQhdLMYtLGMZ81HpuausEvYnc8++ICB3V6mMWEyOeR5bKxP1TXs+jKIlCVWfAQBCjTikRZAuGtUHPQNKMf2egSWFVRG+8cKV+SBgDUetILP6ocu77W+Cj1L/7XiLarQX8gs8Ymhvt2Vk0jFlwtWc3ZTvb/VZLfPQNnbGvuQxKoxDdCIzOt7NQAAAAAAAAAAAAAAAQAAAApiaWxseS1jZXJ0AAAACQAAAAViaWxseQAAAAAAAAAA//////////8AAAAAAAAAggAAABVwZXJtaXQtWDExLWZvcndhcmRpbmcAAAAAAAAAF3Blcm1pdC1hZ2VudC1mb3J3YXJkaW5nAAAAAAAAABZwZXJtaXQtcG9ydC1mb3J3YXJkaW5nAAAAAAAAAApwZXJtaXQtcHR5AAAAAAAAAA5wZXJtaXQtdXNlci1yYwAAAAAAAAAAAAACFwAAAAdzc2gtcnNhAAAAAwEAAQAAAgEA57aSrYecHkvXEPQYZ3ZXyld2bJ3CeuD+rmEtbGFYuJ4NjobBTle1fBDULcemeIs2P7rsybixIMVgAhbVRw9Dkm+8yXJXaUib64m5AkPtDrwPZh6ipxAn6EuuRc0bUdw8JGrLk6j4Pb2OClHAI9SEvvhYgdE+2bf2xMvpI63g4elBb7Z0SzKWdPnBbyCDmdQTcC5Fz/8sYzrx/MAhqwZ1kFLr2OahsajpjuJJnL7K7rL+JN2OiBj9eh4Of2IX15FMFV8LHdLsOxn9E3oOQQxNfkXyT/qcyd2SwqmJHEM6AyQuWkWKowtjrLX410VJMUFw8w9q7oSW+Wr6UkTZXg74W+jfRjOBt6CO3I+glEdpeJAV+odaY8Sc3vv+sSRNo+O0Yzskq1voxpw5xQIdFZIzQmN5T5M8bNFVoM0dZ61MiC8gYWAj3w/kUF7hjb4PW2MTZSxP11J7sR9vdV86bos9r1kad6QEiowkJBRYTGPOZZEj0X/1JjrETahFXjV8696Reub5vPzVcizMKQ93iJNMNf/QX4Xrb3FVxmA0HyM6iXM/oLWCQ4A4hdnzONS4e6hURw9BUF7094/dea7Agaw9V97YDXyBBUx80kfwMWdGMYIvRlWJiGt3KrgPERJsx7CCJfPWz3A/Gbgrk9cZXXB5K2gJpQAWv1UpZCWwc2gQqu0AAAIUAAAADHJzYS1zaGEyLTUxMgAAAgCmUjbVFDwtQV6DSV1LO5Xj2MQKaI739Z06PfTadFQ2Ppcp7cmGdQ0AD2HiqiwDcYW4qYpTLB+P4vavExIPrSyX40fTCgDOXJc7SvXQlnbKkNcuQNn9b6EfLMGS+wXdjDHR5rdUTYrsz3KLhJznahECT2ZQdeuNGKcDPFjU38XmslTeSeExjTu3bxY3bqlAnXMD3k7KG/q7H4sa3um9tV/2iycRxTk1GWk2thifMkxVKNmvPSruqI7EKo52dTkYVVM0OgwHK9etaKs8cc4FFyuaPPUBVO76hwYUoUbdfMbIOAQ7J1mnjTB+HkPhbz4/eu9bhcwXII1I/lxN4+NDL+7CQls7+WjrqlR8zU296/lN/xN+ZK/e8cZVaDUxcM0/1YugXTxnwGeosHd2Jj64p7w/F8wKgzy6EEEvshBANH+DsDm47RHzf8lV3JdonLs9J8byejP/6A7pvU9tqhIL7Gb0anKyxyC7BT9S361AVFo7s1ANH8j8pVc7hU2q4D9OPWPx2l5nO9WOTuzdrq6BtEw0s5qaRsXguJVcW5ba7wIPCgu5LRDGNW6Hb244Lce1HZo+aSz8JbAJXpb/TM9eWz3Z2xjnmkJrhtmvck7Ykix0Mt6j9bLGRv5pSLZIYLfVHJcdgsnQPT06c/1Cyo5gYTeeflrqK4O3TIyoBurA0rVtew== billy@example.com diff --git a/tests/load/filesystem_sftp/bootstrap/billy_rsa.pub b/tests/load/filesystem_sftp/bootstrap/billy_rsa.pub new file mode 100644 index 0000000000..8b74bdb00b --- /dev/null +++ b/tests/load/filesystem_sftp/bootstrap/billy_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC3q9LEVSygyZ+1JzayfvJ38ao3VC4PYOULN2n4w0XZrAp7glcL7284oPBhaTUNTVnavjTSFQWmwKiVes7uQ0kp0s1nDu1RwjC6xdRahuHfNPRUKmianJwkjXKVkdNftxLC5JfjdVE2ayPTj2xrBO3Bv7nJRGxm5zx8tFfjHQRiNuqs/hPES+6a9d/a+M9CD6V3hAIBJNY2KM3RoFFX5+d8JeWSS370+HKpRgWYt93/8lyUSzkkTI6axfry1RPH3Rc0oLh5/S9gOdCwxMO6Mc7COVr1lNUy+FcGDXRlR36uZtaXAP2AKIiovlhmD+Gwn1ARViv7TeESr6qcECpFPC8VdokcZRU2Dt4KQTfDhEVKpTsUjBqD2docFC9tbQsswAikmpE9DzUnQsm3Fc+yJf7G3g5qH6OBYwBe0qR/PNl+74FENngX9OpGPvGA+vCwKAgUTqfc+mCcFFlvUhzNDRUW4Zc7aS8Kkf1snlFvwwQnJnXP8aqJqAF8OFQhdLMYtLGMZ81HpuausEvYnc8++ICB3V6mMWEyOeR5bKxP1TXs+jKIlCVWfAQBCjTikRZAuGtUHPQNKMf2egSWFVRG+8cKV+SBgDUetILP6ocu77W+Cj1L/7XiLarQX8gs8Ymhvt2Vk0jFlwtWc3ZTvb/VZLfPQNnbGvuQxKoxDdCIzOt7NQ== billy@example.com diff --git a/tests/load/filesystem_sftp/bootstrap/bobby_rsa b/tests/load/filesystem_sftp/bootstrap/bobby_rsa new file mode 100644 index 0000000000..1ee49f5776 --- /dev/null +++ b/tests/load/filesystem_sftp/bootstrap/bobby_rsa @@ -0,0 +1,50 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABB98Gk5Ak +IB7cgFibVwRnadAAAAGAAAAAEAAAIXAAAAB3NzaC1yc2EAAAADAQABAAACAQD2sf6SyXup +5tWG/cqSTtORJJGM1teEnDpyupHbUP1LHXvq5nHJ0g82YX9v3wUJ3Nkd6ZMbh+bs37BCaQ +zR2oDWs3kIhsx/xesg03N6tdVMpXFw3tgSceSsXe31YS5rUVNGYIUVtcP3xFRQvLG41X0/ +5GEGS+yiicq+LfqyMnE6Np+/FGaCFzuW1nE8hrR4B/YoLKJs8e4kKOs8Jsj8mCCAXvAk+S +NDZLbVhx5pQcybUSNYqC4NExn5EEj4xK71vHa4oA3gtgeubKXVr04fk5C8maku/tr1P63M +UKtuvTtZBmNH+RVs9i9IwsF8VSMBSMyTMuOaa1Tsa3FIwnGZpZ1PcfTBG7CJPMu4drMBSw +5y0DiAY8mdGvdklz1esG0DldAZx5Blfvw6XU16588e3zQKNiApNwQHlvJJEb0Gb7rFw/Tt +HhCT8MtpBqIf+gRMaFhajbgkQI6CDABp6RgmpqcI6xDFwvkQ+JFveFNFy6zXVrd+oYgz16 +owDpEzfxTYp0mrhivpa0dkoKqgjIU8VMuQfegoxQiJUi+mW9IqrmzcGwAlRTzBcT+IJ9Jz +mfZxuZQshWIQe05nPMqCJqpSukg6tN+Fl0kVF8swz6pNjEIcAryIuHlqOxSyQz8pqxUPBS +PqAEcY0i+0VeMfweQ2TNHdtT3IYzOZrT79r/Ba6Jhy6wAAB1DPvTW0+gfDxLsGAokxLfHY +YtTqqu6siXxI1YPVS57m3HV3iL4PZ3yCcofMFjHT3R+QfSlJxmjg4cHLKCe2P4h10fMxSa +1T+H/NHJvh4ZXt2PfCGXKcXl65t/d5KnM1LSCWHxAXH+u8gOL8giRXljzzjjEl2xFchiYn +zf+pGaCBgfbd1232lePkeKHQSyR6gSOe5t15BwUVRoOKXIrFgpyf+a5i4sbOqBr464wcFd +cLrFcXMmtArNzWLrtBfyFgtyv2KqcdQ7PhUh4JZoNXKSI+28P8sj8xoe5PCCUnk9JlHsEe +j09mzPSqrfHm6JcEuuR/685hhTvlB3Wo0q0dc0AHIDNHQjL0WmKyDWGc321A+QquJdhkYb +v/BuDzrSuSA4tTrMfS84LNFvDtB0NtxfCz/doqiqTaMHre1DrA348Cvzrs53SQjiiT8PUC +SI5mO6u+XY5wyi80knGhJdOYZSwB/m8BuGZJlzRrR4hkbCBtrnwVd/R7jlIdobqJm5Y4+i +iocCGCAvEsKUz5RYZdJiCC85PhsUOtIdFeaCltrTNgmpAKO0GQdI4mYRUOrcXoaM6Y/eMi +zkbcDeitvGcT2YYkflmwxg/7G46sRbICgf8s+lcRix0P6grSvAINGdNBE3yz9YfeNKJh43 +BSQQxulk7ZM0juNqbMl/gGBikncsnx/aVusBzqo3jxZ0g2X2nn1rS0So13Fc54I1957YME +L0u3xY9nttkCY+TC5q+DuwjRZnu9uUe7qPSNpcI4L7SnU6YpW8qdkYf00Lp/CVAks8YjVx +1b0+FiM9CIYAQwQEcEGCHhi24IYwVLvvL+sXZZYRubgCOy74bxJKDZ173SJySvXdjz2d89 +gasbagNdvLP50WPm+b3KaCyUwhyHez0r23LnCAHfHi3MRO3FJPQr9/F5XsHDdpChWKngdq +juKQNtTuqwXlaSr+M8Sr7IMnnpzBeqsfldT223+kwmeWdmvXeR/Xcfw+8pexn2u8GL2jW+ +T5P7zaosKXHQwVUxOOvwQkzo7ldE3Dwj+SC//OfZtMWampuX2xoYbqxQEkAUGO+8g/FcA0 +E3TU4LlhbUbLZj2EA2nZSb78P9N9Uf3OzP9Cjl/g8B6jpDv/jzwXugONMZvANle8mwq4PV +2hRsdo/Y3IQ5eBO2z1sONbN0oluvdfwNVyS6kWIDT/EVEseeNUTKr0EsgoKNswxw/ZeTdT +fTqV+JSmiJSyy3pBaBCDs6w6Xo2wG1AZI7ggtb5zf/QDQqCgL1cvfneQx6de9aFmozt+Bu +DlmBchWcuSEZMrWaYsSuPNy2DgSMZE1UyTbnHwl8fC9N0etwvTD+C2L19BIUt+AsIUZ08R +WmBEYY5aeAlmZnNSVgakHQpyD2EY/jB89l8n2yK74Ortc7e1qbJtEn5KOd5LywjI1N+JLT +M3yUvz6+TFvhHi/mmgb05tcz60QR9CVl61zliVlV+TdP3smmypBjLhxa7a/FmRLGuSU0V6 +mOvpmALMrfRQRR3V5Mkq4kJsR6khBXwljhwtXveT67o/A9HygmAfG7+nAgPFHlCkHzPB28 +HAXwguiSrfUpyIzCTMvVE1mpt9dLN9RYtD1rcVfGBwUmZmSz1jP36pJZi1/84/8TeL3wjK +iJkJZ+UFhMkz+E2m25e/eHmeOmfxHCvPKqQAIzQLC/yR1wkp3hZIhk/6/ImJmGtjoS7/O3 +I3c998HSjQgP2OE6spr7J2YuLfUuz4QOFTs757pzgnfQNIrxioqmLb8dOGCfYhDeW5cEz8 +CE9piaoXXSFpQCsBwer+KFiyOdvNIeNDQBoruElS4KTO/qpYmDkOqzCywpUQCvmx6umqFd +5XaibupiP87chS68oyR3Fs/Ga+qsZ94CFi+6WYTT4GRug5flGtfZP5BeRN4O5AtICr/1pR +7tbw7lgwI1PN1G6jxMSrmstIjOA2vn9KxeRxpPLw+I5SjBZsyfjzjoCsZqTy9gvpnxf5Cu +Il/vgIZXNh3WC8Ypl7zvXsesiISd/7EiHPtncDy9dPk0XMVMfMUxTzWY7tle/xma8atRv7 +vf8w/XfP192OmpYf19+YwNjjUOksuT3WC50NbYAtYasIqJQtxC+2XgaCbf2bheZ01HBEO5 +qOqB+6o2KPz2cxjZIxRsyeX75VMETjPCQvYv9DIMRN4UP14VJEbLbVFDB9HOYpCirJrUC8 +NjUDdlTouyzx2F5Bw4mDnBOKRYuDtIwh7FtkDDCP7Fu9bIzeK+8Tpjn/WkIv600rKmrnZ9 ++a2Fy2LKIKJjyx+oISFiE52kE+rUVW/KKajHdxVZVbdOoHXzst5ZINFJ6uVaPDcyErsIoq +bbsRY8B3WwaU8Qa4AKUiqfTahZ22ducEyiDYs1apLZuE1HWKORXgWPMtdLPAE7+E/zo5ss +d7g+ddymwCBEV6q7p6Nw2f7HlTO++cvkB0EQDmrpWEcoRkSELTBQ6/SND90CX4yo2Rh6qq +Qi1l9MV2/ft4KKZsTqSlVV+po= +-----END OPENSSH PRIVATE KEY----- diff --git a/tests/load/filesystem_sftp/bootstrap/bobby_rsa.pub b/tests/load/filesystem_sftp/bootstrap/bobby_rsa.pub new file mode 100644 index 0000000000..1f9ef6f504 --- /dev/null +++ b/tests/load/filesystem_sftp/bootstrap/bobby_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQD2sf6SyXup5tWG/cqSTtORJJGM1teEnDpyupHbUP1LHXvq5nHJ0g82YX9v3wUJ3Nkd6ZMbh+bs37BCaQzR2oDWs3kIhsx/xesg03N6tdVMpXFw3tgSceSsXe31YS5rUVNGYIUVtcP3xFRQvLG41X0/5GEGS+yiicq+LfqyMnE6Np+/FGaCFzuW1nE8hrR4B/YoLKJs8e4kKOs8Jsj8mCCAXvAk+SNDZLbVhx5pQcybUSNYqC4NExn5EEj4xK71vHa4oA3gtgeubKXVr04fk5C8maku/tr1P63MUKtuvTtZBmNH+RVs9i9IwsF8VSMBSMyTMuOaa1Tsa3FIwnGZpZ1PcfTBG7CJPMu4drMBSw5y0DiAY8mdGvdklz1esG0DldAZx5Blfvw6XU16588e3zQKNiApNwQHlvJJEb0Gb7rFw/TtHhCT8MtpBqIf+gRMaFhajbgkQI6CDABp6RgmpqcI6xDFwvkQ+JFveFNFy6zXVrd+oYgz16owDpEzfxTYp0mrhivpa0dkoKqgjIU8VMuQfegoxQiJUi+mW9IqrmzcGwAlRTzBcT+IJ9JzmfZxuZQshWIQe05nPMqCJqpSukg6tN+Fl0kVF8swz6pNjEIcAryIuHlqOxSyQz8pqxUPBSPqAEcY0i+0VeMfweQ2TNHdtT3IYzOZrT79r/Ba6Jhy6w== bobby@example.com diff --git a/tests/load/filesystem_sftp/bootstrap/ca_rsa b/tests/load/filesystem_sftp/bootstrap/ca_rsa new file mode 100644 index 0000000000..6616331a46 --- /dev/null +++ b/tests/load/filesystem_sftp/bootstrap/ca_rsa @@ -0,0 +1,49 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn +NhAAAAAwEAAQAAAgEA57aSrYecHkvXEPQYZ3ZXyld2bJ3CeuD+rmEtbGFYuJ4NjobBTle1 +fBDULcemeIs2P7rsybixIMVgAhbVRw9Dkm+8yXJXaUib64m5AkPtDrwPZh6ipxAn6EuuRc +0bUdw8JGrLk6j4Pb2OClHAI9SEvvhYgdE+2bf2xMvpI63g4elBb7Z0SzKWdPnBbyCDmdQT +cC5Fz/8sYzrx/MAhqwZ1kFLr2OahsajpjuJJnL7K7rL+JN2OiBj9eh4Of2IX15FMFV8LHd +LsOxn9E3oOQQxNfkXyT/qcyd2SwqmJHEM6AyQuWkWKowtjrLX410VJMUFw8w9q7oSW+Wr6 +UkTZXg74W+jfRjOBt6CO3I+glEdpeJAV+odaY8Sc3vv+sSRNo+O0Yzskq1voxpw5xQIdFZ +IzQmN5T5M8bNFVoM0dZ61MiC8gYWAj3w/kUF7hjb4PW2MTZSxP11J7sR9vdV86bos9r1ka +d6QEiowkJBRYTGPOZZEj0X/1JjrETahFXjV8696Reub5vPzVcizMKQ93iJNMNf/QX4Xrb3 +FVxmA0HyM6iXM/oLWCQ4A4hdnzONS4e6hURw9BUF7094/dea7Agaw9V97YDXyBBUx80kfw +MWdGMYIvRlWJiGt3KrgPERJsx7CCJfPWz3A/Gbgrk9cZXXB5K2gJpQAWv1UpZCWwc2gQqu +0AAAdYWw35zlsN+c4AAAAHc3NoLXJzYQAAAgEA57aSrYecHkvXEPQYZ3ZXyld2bJ3CeuD+ +rmEtbGFYuJ4NjobBTle1fBDULcemeIs2P7rsybixIMVgAhbVRw9Dkm+8yXJXaUib64m5Ak +PtDrwPZh6ipxAn6EuuRc0bUdw8JGrLk6j4Pb2OClHAI9SEvvhYgdE+2bf2xMvpI63g4elB +b7Z0SzKWdPnBbyCDmdQTcC5Fz/8sYzrx/MAhqwZ1kFLr2OahsajpjuJJnL7K7rL+JN2OiB +j9eh4Of2IX15FMFV8LHdLsOxn9E3oOQQxNfkXyT/qcyd2SwqmJHEM6AyQuWkWKowtjrLX4 +10VJMUFw8w9q7oSW+Wr6UkTZXg74W+jfRjOBt6CO3I+glEdpeJAV+odaY8Sc3vv+sSRNo+ +O0Yzskq1voxpw5xQIdFZIzQmN5T5M8bNFVoM0dZ61MiC8gYWAj3w/kUF7hjb4PW2MTZSxP +11J7sR9vdV86bos9r1kad6QEiowkJBRYTGPOZZEj0X/1JjrETahFXjV8696Reub5vPzVci +zMKQ93iJNMNf/QX4Xrb3FVxmA0HyM6iXM/oLWCQ4A4hdnzONS4e6hURw9BUF7094/dea7A +gaw9V97YDXyBBUx80kfwMWdGMYIvRlWJiGt3KrgPERJsx7CCJfPWz3A/Gbgrk9cZXXB5K2 +gJpQAWv1UpZCWwc2gQqu0AAAADAQABAAACAQDf6BPK7c0VlZGoR0fByqDA3U2JXdUNapeQ +KEV9gn/6Pni5LkxjiOvNjHGMH24k89kjMl0X/InlMfU5b5j+xqg0PBdAzmXbThelPntxzf +OoFmAG/TmNmFYH6gHoX2+z5c6UHkLEWDYxaxdzr1WtCfKQm2jjYTCLsHIb7zNFkANMKlta +OQQdmi+7r5CU8uVYY4+5Cm3ZtSnOZapq47plCqPWJ9rRyOOcyq2OreoWNmPgz0Q4mYPezc +87DwpRW5fQbg4IZMRYOHagEdf01zAo+Vt01uB1GqYhhyQHSU496lSHrpHAF40FSr8xqNJi +Kwi0ORmNNmvNOoCtWgu0LeWiN3MTflJFOTVzXD2+GB59BR8O2mE5akNOce3goDcPDz5rYA +onFi3KRqm6rA1pQKGkTVBCzCt4rEOeZ69yaGcNZl7qv4sQw1FOFjKsmOjXW9hVwDuHZU37 +bruCNrEHnFh27KDic0EptKypDC07Q+PmC91gosuGzM3U8fyB08m/YCFAd+WXUQSlIJebxA +SjzneIoA7FBUf8l5Cr9vzaTiq66+Epj2uwMy6EssVmsY8Ey++qOYEwVccTMV5EH0/AdZUh +T7ynjQH81whLYh6CrmxFcdUtvkQeQbGInqQtOVG592CIZM+r/YmyRzuFIxEkT2RWVLJaJL +l0w9qehS3FP7nVs1kgvQAAAQBvxrDXLnGVocaBE4gSVnS3gUd00W6iY44m4aJLwC2PItNF +SAXYhCGtk+x/fE3LE5kyPM2N0Lp2hU2+a4+go3uz8o1WkbQX4vFIv2mUzUyjjBq8bAIHVM +tRjiQaVda8s79d9zpQYhB7hCicpNBR2CMiV4JBeYD46W4ds0cDCOb/gsUgrAOAatCq8B8K +vpepzIm8SNUkhLkTpP9MaekGdXktPyeIrMm7RD/ZBfdu7TiQidNXNErBr3IMLZnUARc9Kz +PWnMOayqPeCpVY+/Dl6jVx9F2QTc9BUcovltluRcT4ov9iPNpPVIOqV6iyU0SxjQGwU3Rl +Rzmf+lUKsiYZCp6CAAABAQD96tImcK0UE5AwSM1B5LSQIfyRNiXVNJXWmHZ4+hd3CztMXf +wZFtQNvxitK+ZD1FqMW3S6Gpy7Qn6BKl0qz+6e4oQdkdJ9wqkxTBaPL1y8rGscTYt/BGZH +ESI0xT3VnkDJNmrX0X5SzSJqFPb82Ei1UYAIypL5xVNc/lNXIrOADEnzjkEbh/AcPyQAjP +PLZI8T1qsv/gWw6bEI2tZWOZDIcPr10THMPvLG6nih7pwENsnI+TSRkyhPNj7nd0ZcNPnk +hGhjL2tFdus6ybZ5Y6UEgMjPISnGaGKhn8KTitn1LHrAKQQQPmeuLynTkDOAEuZ4/pA0J7 +i+4IB/y2uHrEhbAAABAQDpnSCpsKHT+51C1GEWHccowKaMC90O/TuuOBSKT29Xef8+q5Uo +U4lsVTYQTIKzXgtNaGKJLC7RdshIEiTZIT3V/VrPkpKu0jqpeRZCm2nOpabB/1/m0z+pxF +M3hb8svIMwlcw3m6ljbpuyDVHOMKHcRQS2IcalOqTehutlI5nlJsz9BlOmO4Wl0G1b/8dw +B630j7e9vBTuhbLhfjP+AExtU1xCQfkNKVyWVgDYJweWUzBicHKZN6IoTnW3DKpEue10mT +kXJb+MEgW8w4HODogYVvNQDf84NqC9up1zdfYqtcf+slfFcWyC/2NF4413ZnCQGPHTIKke +sqDOI+iF6HxXAAAAG2p1bGlhbkBKdWxpYW5zLUxhcHRvcC5sb2NhbAECAwQFBgc= +-----END OPENSSH PRIVATE KEY----- diff --git a/tests/load/filesystem_sftp/bootstrap/ca_rsa.pub b/tests/load/filesystem_sftp/bootstrap/ca_rsa.pub new file mode 100644 index 0000000000..e39fa19142 --- /dev/null +++ b/tests/load/filesystem_sftp/bootstrap/ca_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDntpKth5weS9cQ9BhndlfKV3ZsncJ64P6uYS1sYVi4ng2OhsFOV7V8ENQtx6Z4izY/uuzJuLEgxWACFtVHD0OSb7zJcldpSJvribkCQ+0OvA9mHqKnECfoS65FzRtR3DwkasuTqPg9vY4KUcAj1IS++FiB0T7Zt/bEy+kjreDh6UFvtnRLMpZ0+cFvIIOZ1BNwLkXP/yxjOvH8wCGrBnWQUuvY5qGxqOmO4kmcvsrusv4k3Y6IGP16Hg5/YhfXkUwVXwsd0uw7Gf0Teg5BDE1+RfJP+pzJ3ZLCqYkcQzoDJC5aRYqjC2OstfjXRUkxQXDzD2ruhJb5avpSRNleDvhb6N9GM4G3oI7cj6CUR2l4kBX6h1pjxJze+/6xJE2j47RjOySrW+jGnDnFAh0VkjNCY3lPkzxs0VWgzR1nrUyILyBhYCPfD+RQXuGNvg9bYxNlLE/XUnuxH291Xzpuiz2vWRp3pASKjCQkFFhMY85lkSPRf/UmOsRNqEVeNXzr3pF65vm8/NVyLMwpD3eIk0w1/9BfhetvcVXGYDQfIzqJcz+gtYJDgDiF2fM41Lh7qFRHD0FQXvT3j915rsCBrD1X3tgNfIEFTHzSR/AxZ0Yxgi9GVYmIa3cquA8REmzHsIIl89bPcD8ZuCuT1xldcHkraAmlABa/VSlkJbBzaBCq7Q== julian@Julians-Laptop.local diff --git a/tests/load/filesystem_sftp/bootstrap/foo_rsa b/tests/load/filesystem_sftp/bootstrap/foo_rsa new file mode 100644 index 0000000000..9ccc98eccb --- /dev/null +++ b/tests/load/filesystem_sftp/bootstrap/foo_rsa @@ -0,0 +1,49 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn +NhAAAAAwEAAQAAAgEAxz/mzyh9N8KvqW5UVMuKo7reYYyUxU2gGYv/StDs+H3j5UQbrhrA +bwV+R0jgfK4pEAQw2EDDWz5r8pc2LOAq9C+mpha+R6oXZuLwZFCnJOro+peFDEkGVyNReq +Sw2JvKFenRiP2IWqQRtv/zfbcC65TrX4C/DNFuGF/uPjZJH8v4rGN+MtAcrQZDi7V4pvAe +FejIeVumlU5rZ6XN1hOID3BrmybiL8cqcyGTVolYbnKdyaPRmjmXgGBK0vMecJhvbWzVlW +7iLnXnaULXFiaKxXlvizxwMampCNMbD6j0+b6DolYDjEj0BRuEWGr2Ox3opqKhWww1eAtj +spY7PpGAoeCq1Mfvn178Bv0sRZykJT4gM4ELOEhmItl6QSFn5kLg8vDRg0j8CHe5scVSVG +3egjYVcNxL/05okSiWIwNLd7Ma5Qyi06BS6JUqqFEDhANIRgMZuztMs1e6IKtiiREeOhf4 +UQLcMsH8r0AYjJPXFTpXvWLamsiAhsL1tGa8XyyxqdJrqsIktrqS4QnCx7uDAPon0wHoRK +YzCwqNmn1Iwf3uepZ7Z0evh3IRRfSI3LaRxfhM0I63z8uPyzx55C3PAkJpmRiPGzFWd7Nk +1P6OgRsjhlSfEK8OwOFXfZLYoRBEYj8yUXRhJu3CdcJViCIJYuhtY38fST3875aTx1KJOq +sAAAdIRs3hz0bN4c8AAAAHc3NoLXJzYQAAAgEAxz/mzyh9N8KvqW5UVMuKo7reYYyUxU2g +GYv/StDs+H3j5UQbrhrAbwV+R0jgfK4pEAQw2EDDWz5r8pc2LOAq9C+mpha+R6oXZuLwZF +CnJOro+peFDEkGVyNReqSw2JvKFenRiP2IWqQRtv/zfbcC65TrX4C/DNFuGF/uPjZJH8v4 +rGN+MtAcrQZDi7V4pvAeFejIeVumlU5rZ6XN1hOID3BrmybiL8cqcyGTVolYbnKdyaPRmj +mXgGBK0vMecJhvbWzVlW7iLnXnaULXFiaKxXlvizxwMampCNMbD6j0+b6DolYDjEj0BRuE +WGr2Ox3opqKhWww1eAtjspY7PpGAoeCq1Mfvn178Bv0sRZykJT4gM4ELOEhmItl6QSFn5k +Lg8vDRg0j8CHe5scVSVG3egjYVcNxL/05okSiWIwNLd7Ma5Qyi06BS6JUqqFEDhANIRgMZ +uztMs1e6IKtiiREeOhf4UQLcMsH8r0AYjJPXFTpXvWLamsiAhsL1tGa8XyyxqdJrqsIktr +qS4QnCx7uDAPon0wHoRKYzCwqNmn1Iwf3uepZ7Z0evh3IRRfSI3LaRxfhM0I63z8uPyzx5 +5C3PAkJpmRiPGzFWd7Nk1P6OgRsjhlSfEK8OwOFXfZLYoRBEYj8yUXRhJu3CdcJViCIJYu +htY38fST3875aTx1KJOqsAAAADAQABAAACAFjd6WXP2zl5fbuF19sSBT3NZM4BU4FEg/mg +9TY7RNX34CMrY2UdzWI3AwFsQaOaUfowxFBPYlJZ3u+N/b26Ja5PanZ9glSYSmO7KBi12D +ahB1RtLAw0rb1DpV2cArw5j8KCTNBas+wpbTU/pywU6hqEiw5Hb+6Zog8BClN5BthFsx3A +KlMjewa42nt/btaWFfUTpAZsmDnThhfuXYXzpCWusG+8wfkpTYeYHAzmqShpunJqvFubAD +VjvTuk75ishFY7ym8hy4OJVrMd+qyIeDBnXxas2CVuVFP5RAKSCuEw1akbz+1LxHpasYZ1 +/miTiGZ59pmTMm3eNpM6aiYX41SFsx7plNIbHG/BYbardc9ZVrZZO/fyh2LHzQ8vGALxHP +ydIUQVKmcWXjFwBVwCZZc83FVyPfdSH3wxK/6MiAAWNDw564d3lrMHZ94n5EX3dKj/+mLB +okG2FTxhDhYGwxCcOoE0YCHHBfxOu08RRpXYLvYPFzrhuHKslJCVsM6BgebBSxlhcJDIZa +NOloE5COuh95byGmMr5DYnimsEXSpJFz4nzscFF/gkoTn1juDOTmhO9D8blFLHOtwCJ3ix +juASf6ydiHTutGHAzOIMXC8K3Tci9rUQtoNMStUypBzAFx7s6Cv62/wqCsWno/Ne/G9eFB +U3bAtyVHGIYuvRujBBAAABAQDrZ8rR3kBxbqaEN9oy9xKn4i7UiB/LRXPjOdmFIH7Lo51g +1yQ+jWtEj6nOsFHw+yhei8QdFyI8xxf7+H6PZY6anGpr48hfuE86bcPFT/6VetfjkOSaPb +mfTqckhH0Cye1AH5nmrpgr75E3eVZcg4dt7s8E7R3XIi+qUVYN9ERfwO6AzYS6A3AM144c +u3bG5WxcbhgFdGy6iB26B9UO6+VYvu7HMn0MP+dUU3sdsV+rTQJHT5zA5oIWeRL5BSxnQk +NpuacEyFXiAAMqrBZp97bO0e9dymoMFvbznxgPQcla003PTizLMOnE01USqd4jcE3+F43S +eAdu0k69Hl0tzwv8AAABAQD0l7N0XeJPzcveCB7FSSqTVHxztCpcWdRhul1kQs91Zp/0sN +lwcyAE9ADjC5zsyeAx9Q5TF94HUQ3iuG/aICd6ItOeK06X/r1e3/ole6K7lbSzDnLZ+Fi+ +IvVXCCjRmXMI/m+4+vIXs6y9SZTxmKNm47Wpfd8fumYzfSBL54o5AnKvUYYGiwwuAlKMz1 +rgvRP4ZrzQJZitCh86/jCgRwCDMniu5GGUDXxhvNPoQqeO1ezLV0v9K1SZvd5U0uXnKWop +SLtx2K4s+yyjpxnWyIZmAwjh5paHjLKegXTYskg4n/3fBW9nEJVrFflLVWdzp+xWsz+u3P +sQ3n0efAZPjOWbAAABAQDQitS0R2KAOci5f8LiGy2KtJipcS2jvzBCZX9DqMInfpU/cY92 +bAn8NgY9G8jChloAu2smHo3Fx6LL4ZdFNTjfIBbFyZsuSPhy9czpxWF733mteyuMW6n2jv +e04YoWRp4uh9YxbtpFx24x0RIu64NM69/N2E94eFEJUhpG7NPPgoL41jEqJVJCPVJsQnFT +RCReMb5D9zWYlKVK4xnuB/NqgD+j5iLMHK5hS6Wt0b0olNoOmTlj7IUK62sQyPelU3I3Sy +hmZquXqCILq/rMbJBo5NjhoodvzSFYw+jDvLq4rK+XGL/DgGV080oemXTAP73Er8tBdJq8 +iv8eCatJLFgxAAAAD2Zvb0BleGFtcGxlLmNvbQECAw== +-----END OPENSSH PRIVATE KEY----- diff --git a/tests/load/filesystem_sftp/bootstrap/foo_rsa.pub b/tests/load/filesystem_sftp/bootstrap/foo_rsa.pub new file mode 100644 index 0000000000..212cbb4d7e --- /dev/null +++ b/tests/load/filesystem_sftp/bootstrap/foo_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDHP+bPKH03wq+pblRUy4qjut5hjJTFTaAZi/9K0Oz4fePlRBuuGsBvBX5HSOB8rikQBDDYQMNbPmvylzYs4Cr0L6amFr5Hqhdm4vBkUKck6uj6l4UMSQZXI1F6pLDYm8oV6dGI/YhapBG2//N9twLrlOtfgL8M0W4YX+4+Nkkfy/isY34y0BytBkOLtXim8B4V6Mh5W6aVTmtnpc3WE4gPcGubJuIvxypzIZNWiVhucp3Jo9GaOZeAYErS8x5wmG9tbNWVbuIudedpQtcWJorFeW+LPHAxqakI0xsPqPT5voOiVgOMSPQFG4RYavY7HeimoqFbDDV4C2Oyljs+kYCh4KrUx++fXvwG/SxFnKQlPiAzgQs4SGYi2XpBIWfmQuDy8NGDSPwId7mxxVJUbd6CNhVw3Ev/TmiRKJYjA0t3sxrlDKLToFLolSqoUQOEA0hGAxm7O0yzV7ogq2KJER46F/hRAtwywfyvQBiMk9cVOle9YtqayICGwvW0ZrxfLLGp0muqwiS2upLhCcLHu4MA+ifTAehEpjMLCo2afUjB/e56lntnR6+HchFF9IjctpHF+EzQjrfPy4/LPHnkLc8CQmmZGI8bMVZ3s2TU/o6BGyOGVJ8Qrw7A4Vd9ktihEERiPzJRdGEm7cJ1wlWIIgli6G1jfx9JPfzvlpPHUok6qw== foo@example.com diff --git a/tests/load/filesystem_sftp/docker-compose.yml b/tests/load/filesystem_sftp/docker-compose.yml new file mode 100644 index 0000000000..a714219146 --- /dev/null +++ b/tests/load/filesystem_sftp/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3.8' + +services: + + sftpserver: + build: + context: bootstrap + dockerfile: Dockerfile + image: sftpserver:latest + networks: + - sftpserver + ports: + - "2222:22" + volumes: + - ../../common/storages/samples:/home/foo/sftp/data/standard_source/samples + - ../../common/storages/samples:/home/bobby/sftp/data/standard_source/samples + - ../../common/storages/samples:/home/billy/sftp/data/standard_source/samples + +networks: + sftpserver: + name: sftpserver diff --git a/tests/load/filesystem_sftp/test_filesystem_sftp.py b/tests/load/filesystem_sftp/test_filesystem_sftp.py new file mode 100644 index 0000000000..32869ee56e --- /dev/null +++ b/tests/load/filesystem_sftp/test_filesystem_sftp.py @@ -0,0 +1,174 @@ +import os +import pytest +import fsspec +import dlt + +from dlt.common.json import json +from dlt.common.configuration.inject import with_config +from dlt.common.storages import FilesystemConfiguration, fsspec_from_config +from dlt.destinations.impl.filesystem.filesystem import FilesystemClient + +from tests.load.utils import ALL_FILESYSTEM_DRIVERS + +if "sftp" not in ALL_FILESYSTEM_DRIVERS: + pytest.skip("sftp filesystem driver not configured", allow_module_level=True) + + +@with_config(spec=FilesystemConfiguration, sections=("sources", "filesystem")) +def get_config(config: FilesystemConfiguration = None) -> FilesystemConfiguration: + return config + + +def get_key_path(user: str = "foo") -> str: + current_dir = os.path.dirname(os.path.abspath(__file__)) + return os.path.join(current_dir, f"bootstrap/{user}_rsa") + + +def files_are_equal(file1_path, file2_path): + try: + with open(file1_path, "r", encoding="utf-8") as f1, open( + file2_path, "r", encoding="utf-8" + ) as f2: + return f1.read() == f2.read() + except FileNotFoundError: + return False + + +def is_ssh_agent_ready(): + try: + # Never skip tests when running in CI + if os.getenv("CI"): + return True + + # Check if SSH agent is running + ssh_agent_pid = os.getenv("SSH_AGENT_PID") + if not ssh_agent_pid: + return False + + # Check if the key is present and matches + id_rsa_pub_path = os.path.expanduser("~/.ssh/id_rsa") + bobby_rsa_pub_path = os.path.expanduser(get_key_path("bobby")) + if not os.path.isfile(id_rsa_pub_path) or not os.path.isfile(bobby_rsa_pub_path): + return False + + return files_are_equal(id_rsa_pub_path, bobby_rsa_pub_path) + except Exception: + return False + + +@pytest.fixture(scope="module") +def sftp_filesystem(): + fs = fsspec.filesystem( + "sftp", host="localhost", port=2222, username="foo", key_filename=get_key_path() + ) + yield fs + + +def test_filesystem_sftp_server(sftp_filesystem): + test_file = "/data/countries.json" + input_data = { + "countries": [ + {"name": "United States", "code": "US"}, + {"name": "Canada", "code": "CA"}, + {"name": "Mexico", "code": "MX"}, + ] + } + + fs = sftp_filesystem + try: + with fs.open(test_file, "w") as f: + f.write(json.dumps(input_data)) + + files = fs.ls("/data") + assert test_file in files + + with fs.open(test_file, "r") as f: + data = json.load(f) + assert data == input_data + + info = fs.info(test_file) + assert "mtime" in info + + finally: + fs.rm(test_file) + + +def test_filesystem_sftp_pipeline(sftp_filesystem): + import posixpath + import pyarrow.parquet as pq + + os.environ["DESTINATION__FILESYSTEM__BUCKET_URL"] = "sftp://localhost/data" + os.environ["DESTINATION__FILESYSTEM__CREDENTIALS__SFTP_PORT"] = "2222" + os.environ["DESTINATION__FILESYSTEM__CREDENTIALS__SFTP_USERNAME"] = "foo" + os.environ["DESTINATION__FILESYSTEM__CREDENTIALS__SFTP_PASSWORD"] = "pass" + + @dlt.resource() + def states(): + yield [{"id": 1, "name": "DE"}, {"id": 2, "name": "AK"}, {"id": 3, "name": "CA"}] + + pipeline = dlt.pipeline(destination="filesystem", dataset_name="test") + pipeline.run([states], loader_file_format="parquet") + + client: FilesystemClient = pipeline.destination_client() # type: ignore[assignment] + data_glob = posixpath.join(client.dataset_path, "states/*") + data_files = client.fs_client.glob(data_glob) + assert len(data_files) > 0 + + fs = sftp_filesystem + with fs.open(data_files[0], "rb") as f: + rows = pq.read_table(f).to_pylist() + result_states = [r["name"] for r in rows] + + expected_states = ["DE", "AK", "CA"] + assert sorted(result_states) == sorted(expected_states) + + +def run_sftp_auth(user, password=None, key=None, passphrase=None): + env_vars = { + "SOURCES__FILESYSTEM__BUCKET_URL": "sftp://localhost", + "SOURCES__FILESYSTEM__CREDENTIALS__SFTP_PORT": "2222", + "SOURCES__FILESYSTEM__CREDENTIALS__SFTP_USERNAME": user, + } + + if password: + env_vars["SOURCES__FILESYSTEM__CREDENTIALS__SFTP_PASSWORD"] = password + if key: + env_vars["SOURCES__FILESYSTEM__CREDENTIALS__SFTP_KEY_FILENAME"] = get_key_path(user) + if passphrase: + env_vars["SOURCES__FILESYSTEM__CREDENTIALS__SFTP_KEY_PASSPHRASE"] = passphrase + + os.environ.update(env_vars) + + config = get_config() + fs, _ = fsspec_from_config(config) + assert len(fs.ls("/data/standard_source/samples")) > 0 + + +def test_filesystem_sftp_auth_useranme_password(): + run_sftp_auth("foo", "pass") + + +def test_filesystem_sftp_auth_private_key(): + run_sftp_auth("foo", key=get_key_path()) + + +def test_filesystem_sftp_auth_private_key_protected(): + run_sftp_auth("bobby", key=get_key_path("bobby"), passphrase="passphrase123") + + +# Test requires - ssh_agent with user's bobby key loaded. The commands and file names required are: +# eval "$(ssh-agent -s)" +# cp /path/to/tests/load/filesystem_sftp/bobby_rsa* ~/.ssh/id_rsa +# cp /path/to/tests/load/filesystem_sftp/bobby_rsa.pub ~/.ssh/id_rsa.pub +# ssh-add ~/.ssh/id_rsa +@pytest.mark.skipif( + not is_ssh_agent_ready(), + reason="SSH agent is not running or bobby's private key isn't stored in ~/.ssh/id_rsa", +) +def test_filesystem_sftp_auth_private_ssh_agent(): + run_sftp_auth("bobby", passphrase="passphrase123") + + +def test_filesystem_sftp_auth_ca_signed_pub_key(): + # billy_rsa-cert.pub is automatically loaded too + run_sftp_auth("billy", key=get_key_path("billy")) diff --git a/tests/load/utils.py b/tests/load/utils.py index 1c47291a6c..f443748f8e 100644 --- a/tests/load/utils.py +++ b/tests/load/utils.py @@ -75,6 +75,7 @@ GDRIVE_BUCKET = dlt.config.get("tests.bucket_url_gdrive", str) FILE_BUCKET = dlt.config.get("tests.bucket_url_file", str) R2_BUCKET = dlt.config.get("tests.bucket_url_r2", str) +SFTP_BUCKET = dlt.config.get("tests.bucket_url_sftp", str) MEMORY_BUCKET = dlt.config.get("tests.memory", str) ALL_FILESYSTEM_DRIVERS = dlt.config.get("ALL_FILESYSTEM_DRIVERS", list) or [ @@ -86,6 +87,7 @@ "file", "memory", "r2", + "sftp", ] # Filter out buckets not in all filesystem drivers @@ -97,6 +99,7 @@ ABFS_BUCKET, AZ_BUCKET, GDRIVE_BUCKET, + SFTP_BUCKET, ] WITH_GDRIVE_BUCKETS = [ bucket