From f3c748b8e73fca8d2b2cd11eed3f281671087fb7 Mon Sep 17 00:00:00 2001 From: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Date: Tue, 24 Sep 2024 20:27:19 +0200 Subject: [PATCH] test(source-gcs): add integration test (#45850) Signed-off-by: Artem Inzhyyants --- .../source-gcs/build_customization.py | 24 ++ .../configs/config_integration_avro.json | 18 ++ .../configs/config_integration_csv.json | 18 ++ .../configs/config_integration_jsonl.json | 18 ++ .../configs/config_integration_parquet.json | 18 ++ .../configured_catalog_avro.json | 35 +++ .../configured_catalog_csv.json | 260 ++++++++++++++++++ .../configured_catalog_jsonl.json | 47 ++++ .../configured_catalog_parquet.json | 95 +++++++ .../source-gcs/integration_tests/conftest.py | 80 ++++++ .../integration_tests/docker-compose.yaml | 10 + .../sample-bucket-avro/weather-snappy.avro | Bin 0 -> 330 bytes .../sample-bucket-csv/test_folder.zip | Bin 0 -> 1522 bytes .../test_folder_example_1.csv | 2 + .../test_folder_example_2.csv | 2 + .../test_folder_test_data_1.jsonl | 25 ++ .../test_folder_test_data_1.jsonl.gz | Bin 0 -> 1800 bytes .../sample-bucket-parquet/weather.parquet | Bin 0 -> 21006 bytes .../integration_tests/integration_test.py | 39 +++ .../source-gcs/integration_tests/utils.py | 24 ++ .../connectors/source-gcs/metadata.yaml | 3 +- .../connectors/source-gcs/poetry.lock | 245 ++++++++++------- .../connectors/source-gcs/pyproject.toml | 3 +- .../source-gcs/unit_tests/conftest.py | 1 + .../source-gcs/unit_tests/test_config.py | 1 + .../source-gcs/unit_tests/test_cursor.py | 22 +- .../source-gcs/unit_tests/test_helpers.py | 1 - .../source-gcs/unit_tests/test_run.py | 7 +- .../source-gcs/unit_tests/test_source.py | 5 +- docs/integrations/sources/gcs.md | 75 ++--- 30 files changed, 926 insertions(+), 152 deletions(-) create mode 100644 airbyte-integrations/connectors/source-gcs/build_customization.py create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/configs/config_integration_avro.json create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/configs/config_integration_csv.json create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/configs/config_integration_jsonl.json create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/configs/config_integration_parquet.json create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_avro.json create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_csv.json create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_jsonl.json create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_parquet.json create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/docker-compose.yaml create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-avro/weather-snappy.avro create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-csv/test_folder.zip create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-csv/test_folder_example_1.csv create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-csv/test_folder_example_2.csv create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-jsonl/test_folder_test_data_1.jsonl create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-jsonl/test_folder_test_data_1.jsonl.gz create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-parquet/weather.parquet create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/integration_test.py create mode 100644 airbyte-integrations/connectors/source-gcs/integration_tests/utils.py diff --git a/airbyte-integrations/connectors/source-gcs/build_customization.py b/airbyte-integrations/connectors/source-gcs/build_customization.py new file mode 100644 index 000000000000..50d1ab1c6500 --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/build_customization.py @@ -0,0 +1,24 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + """ + Docker compose is required to run the integration tests so we install Docker on top of the base image. + """ + return ( + base_image_container.with_exec(["sh", "-c", "apt-get update && apt-get install -y curl jq"]) + # Download install-docker.sh script + .with_exec(["curl", "-fsSL", "https://get.docker.com", "-o", "/tmp/install-docker.sh"]) + # Run the install-docker.sh script with a pinned Docker version + .with_exec(["sh", "/tmp/install-docker.sh", "--version", "26.0"]) + # Remove the install-docker.sh script + .with_exec(["rm", "/tmp/install-docker.sh"]) + ) diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/configs/config_integration_avro.json b/airbyte-integrations/connectors/source-gcs/integration_tests/configs/config_integration_avro.json new file mode 100644 index 000000000000..f573fe629bb5 --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/configs/config_integration_avro.json @@ -0,0 +1,18 @@ +{ + "bucket": "sample-bucket-avro", + "service_account": "{}", + "streams": [ + { + "name": "test_avro", + "globs": ["**/*.avro"], + "validation_policy": "Emit Record", + "format": { + "filetype": "avro" + } + } + ], + "credentials": { + "service_account": "{}", + "auth_type": "Service" + } +} diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/configs/config_integration_csv.json b/airbyte-integrations/connectors/source-gcs/integration_tests/configs/config_integration_csv.json new file mode 100644 index 000000000000..1a2fd2f18699 --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/configs/config_integration_csv.json @@ -0,0 +1,18 @@ +{ + "bucket": "sample-bucket-csv", + "service_account": "{}", + "streams": [ + { + "name": "test_csv", + "globs": ["**/*.csv"], + "validation_policy": "Emit Record", + "format": { + "filetype": "csv" + } + } + ], + "credentials": { + "service_account": "{}", + "auth_type": "Service" + } +} diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/configs/config_integration_jsonl.json b/airbyte-integrations/connectors/source-gcs/integration_tests/configs/config_integration_jsonl.json new file mode 100644 index 000000000000..ecb9f44a6e44 --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/configs/config_integration_jsonl.json @@ -0,0 +1,18 @@ +{ + "bucket": "sample-bucket-jsonl", + "service_account": "{}", + "streams": [ + { + "name": "test_jsonl", + "globs": ["**/*.jsonl"], + "validation_policy": "Emit Record", + "format": { + "filetype": "jsonl" + } + } + ], + "credentials": { + "service_account": "{}", + "auth_type": "Service" + } +} diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/configs/config_integration_parquet.json b/airbyte-integrations/connectors/source-gcs/integration_tests/configs/config_integration_parquet.json new file mode 100644 index 000000000000..4a8ae44aad54 --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/configs/config_integration_parquet.json @@ -0,0 +1,18 @@ +{ + "bucket": "sample-bucket-parquet", + "service_account": "{}", + "streams": [ + { + "name": "test_parquet", + "globs": ["**/*.parquet"], + "validation_policy": "Emit Record", + "format": { + "filetype": "parquet" + } + } + ], + "credentials": { + "service_account": "{}", + "auth_type": "Service" + } +} diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_avro.json b/airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_avro.json new file mode 100644 index 000000000000..eea4dc358362 --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_avro.json @@ -0,0 +1,35 @@ +{ + "streams": [ + { + "stream": { + "name": "test_avro", + "json_schema": { + "type": "object", + "properties": { + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + "station": { + "type": ["null", "string"] + }, + "time": { + "type": ["null", "integer"] + }, + "temp": { + "type": ["null", "integer"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"], + "is_resumable": true + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_csv.json b/airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_csv.json new file mode 100644 index 000000000000..550861512fce --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_csv.json @@ -0,0 +1,260 @@ +{ + "streams": [ + { + "stream": { + "name": "test_csv", + "json_schema": { + "type": "object", + "properties": { + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + "TOK": { + "type": ["null", "string"] + }, + "UPDATE": { + "type": ["null", "string"] + }, + "DATE": { + "type": ["null", "string"] + }, + "SHOT": { + "type": ["null", "string"] + }, + "TIME": { + "type": ["null", "string"] + }, + "AUXHEAT": { + "type": ["null", "string"] + }, + "PHASE": { + "type": ["null", "string"] + }, + "STATE": { + "type": ["null", "string"] + }, + "PGASA": { + "type": ["null", "string"] + }, + "PGASZ": { + "type": ["null", "string"] + }, + "BGASA": { + "type": ["null", "string"] + }, + "BGASZ": { + "type": ["null", "string"] + }, + "BGASA2": { + "type": ["null", "string"] + }, + "BGASZ2": { + "type": ["null", "string"] + }, + "PIMPA": { + "type": ["null", "string"] + }, + "PIMPZ": { + "type": ["null", "string"] + }, + "PELLET": { + "type": ["null", "string"] + }, + "RGEO": { + "type": ["null", "string"] + }, + "RMAG": { + "type": ["null", "string"] + }, + "AMIN": { + "type": ["null", "string"] + }, + "SEPLIM": { + "type": ["null", "string"] + }, + "XPLIM": { + "type": ["null", "string"] + }, + "KAPPA": { + "type": ["null", "string"] + }, + "DELTA": { + "type": ["null", "string"] + }, + "INDENT": { + "type": ["null", "string"] + }, + "AREA": { + "type": ["null", "string"] + }, + "VOL": { + "type": ["null", "string"] + }, + "CONFIG": { + "type": ["null", "string"] + }, + "IGRADB": { + "type": ["null", "string"] + }, + "WALMAT": { + "type": ["null", "string"] + }, + "DIVMAT": { + "type": ["null", "string"] + }, + "LIMMAT": { + "type": ["null", "string"] + }, + "EVAP": { + "type": ["null", "string"] + }, + "BT": { + "type": ["null", "string"] + }, + "IP": { + "type": ["null", "string"] + }, + "VSURF": { + "type": ["null", "string"] + }, + "Q95": { + "type": ["null", "string"] + }, + "BEPMHD": { + "type": ["null", "string"] + }, + "BETMHD": { + "type": ["null", "string"] + }, + "BEPDIA": { + "type": ["null", "string"] + }, + "NEL": { + "type": ["null", "string"] + }, + "DNELDT": { + "type": ["null", "string"] + }, + "ZEFF": { + "type": ["null", "string"] + }, + "PRAD": { + "type": ["null", "string"] + }, + "POHM": { + "type": ["null", "string"] + }, + "ENBI": { + "type": ["null", "string"] + }, + "PINJ": { + "type": ["null", "string"] + }, + "BSOURCE": { + "type": ["null", "string"] + }, + "PINJ2": { + "type": ["null", "string"] + }, + "BSOURCE2": { + "type": ["null", "string"] + }, + "COCTR": { + "type": ["null", "string"] + }, + "PNBI": { + "type": ["null", "string"] + }, + "ECHFREQ": { + "type": ["null", "string"] + }, + "ECHMODE": { + "type": ["null", "string"] + }, + "ECHLOC": { + "type": ["null", "string"] + }, + "PECH": { + "type": ["null", "string"] + }, + "ICFREQ": { + "type": ["null", "string"] + }, + "ICSCHEME": { + "type": ["null", "string"] + }, + "ICANTEN": { + "type": ["null", "string"] + }, + "PICRH": { + "type": ["null", "string"] + }, + "LHFREQ": { + "type": ["null", "string"] + }, + "LHNPAR": { + "type": ["null", "string"] + }, + "PLH": { + "type": ["null", "string"] + }, + "IBWFREQ": { + "type": ["null", "string"] + }, + "PIBW": { + "type": ["null", "string"] + }, + "TE0": { + "type": ["null", "string"] + }, + "TI0": { + "type": ["null", "string"] + }, + "WFANI": { + "type": ["null", "string"] + }, + "WFICRH": { + "type": ["null", "string"] + }, + "MEFF": { + "type": ["null", "string"] + }, + "ISEQ": { + "type": ["null", "string"] + }, + "WTH": { + "type": ["null", "string"] + }, + "WTOT": { + "type": ["null", "string"] + }, + "DWTOT": { + "type": ["null", "string"] + }, + "PL": { + "type": ["null", "string"] + }, + "PLTH": { + "type": ["null", "string"] + }, + "TAUTOT": { + "type": ["null", "string"] + }, + "TAUTH": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"], + "is_resumable": true, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_jsonl.json b/airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_jsonl.json new file mode 100644 index 000000000000..5cdfe6afbab6 --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_jsonl.json @@ -0,0 +1,47 @@ +{ + "streams": [ + { + "stream": { + "name": "test_jsonl", + "json_schema": { + "type": "object", + "properties": { + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "valid": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "degrees": { + "type": ["null", "string"] + }, + "birthday": { + "type": ["null", "string"] + }, + "last_seen": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"], + "is_resumable": true + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_parquet.json b/airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_parquet.json new file mode 100644 index 000000000000..4d234523f572 --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_parquet.json @@ -0,0 +1,95 @@ +{ + "streams": [ + { + "stream": { + "name": "test_parquet", + "json_schema": { + "type": "object", + "properties": { + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + "MinTemp": { + "type": ["null", "number"] + }, + "MaxTemp": { + "type": ["null", "number"] + }, + "Rainfall": { + "type": ["null", "number"] + }, + "Evaporation": { + "type": ["null", "number"] + }, + "Sunshine": { + "type": ["null", "string"] + }, + "WindGustDir": { + "type": ["null", "string"] + }, + "WindGustSpeed": { + "type": ["null", "string"] + }, + "WindDir9am": { + "type": ["null", "string"] + }, + "WindDir3pm": { + "type": ["null", "string"] + }, + "WindSpeed9am": { + "type": ["null", "string"] + }, + "WindSpeed3pm": { + "type": ["null", "integer"] + }, + "Humidity9am": { + "type": ["null", "integer"] + }, + "Humidity3pm": { + "type": ["null", "integer"] + }, + "Pressure9am": { + "type": ["null", "number"] + }, + "Pressure3pm": { + "type": ["null", "number"] + }, + "Cloud9am": { + "type": ["null", "integer"] + }, + "Cloud3pm": { + "type": ["null", "integer"] + }, + "Temp9am": { + "type": ["null", "number"] + }, + "Temp3pm": { + "type": ["null", "number"] + }, + "RainToday": { + "type": ["null", "string"] + }, + "RISK_MM": { + "type": ["null", "number"] + }, + "RainTomorrow": { + "type": ["null", "string"] + }, + "weather.parquet?alt": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"], + "is_resumable": true + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py b/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py new file mode 100644 index 000000000000..1fe90e1f1374 --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py @@ -0,0 +1,80 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +import os +import shutil +import time +import uuid +from typing import Any, Mapping + +import docker +import google +import pytest +import source_gcs +from google.auth.credentials import AnonymousCredentials +from google.cloud import storage + +from .utils import get_docker_ip + +LOCAL_GCP_PORT = 4443 + +from urllib.parse import urlparse, urlunparse + + +# Monkey patch generate_signed_url to use media_link in integration tests with local server +def generate_signed_url(self, *args, **kwargs): + parsed_url = urlparse(self.media_link) + new_netloc = parsed_url.netloc.replace("0.0.0.0", get_docker_ip()) + modified_url = urlunparse(parsed_url._replace(netloc=new_netloc)) + self._properties["mediaLink"] = modified_url + return modified_url + + +google.cloud.storage.blob.Blob.generate_signed_url = generate_signed_url + + +# Monkey patch gcs_client to use AnonymousCredentials in integration tests with local server +def _initialize_gcs_client(self): + if self.config is None: + raise ValueError("Source config is missing; cannot create the GCS client.") + if self._gcs_client is None: + self._gcs_client = storage.Client( + credentials=AnonymousCredentials(), + project="test", + client_options={"api_endpoint": f"http://{get_docker_ip()}:{LOCAL_GCP_PORT}"}, + ) + + return self._gcs_client + + +source_gcs.stream_reader.SourceGCSStreamReader._initialize_gcs_client = _initialize_gcs_client + + +@pytest.fixture(scope="session") +def docker_client() -> docker.client.DockerClient: + return docker.from_env() + + +TMP_FOLDER = "/tmp/test_gcs_source" + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup_fixture(docker_client) -> None: + if os.path.exists(TMP_FOLDER): + shutil.rmtree(TMP_FOLDER) + shutil.copytree(f"{os.path.dirname(__file__)}/integration_bucket_data", TMP_FOLDER) + + container = docker_client.containers.run( + image="fsouza/fake-gcs-server", + command=["-scheme", "http"], + name=f"gcs_integration_{uuid.uuid4().hex}", + hostname="gcs-server", + ports={LOCAL_GCP_PORT: ("0.0.0.0", LOCAL_GCP_PORT)}, + detach=True, + volumes={f"{TMP_FOLDER}": {"bind": "/data", "mode": "rw"}}, + ) + time.sleep(5) + + yield + + container.kill() + container.remove() + diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/docker-compose.yaml b/airbyte-integrations/connectors/source-gcs/integration_tests/docker-compose.yaml new file mode 100644 index 000000000000..874c1081ef7c --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/docker-compose.yaml @@ -0,0 +1,10 @@ +services: + fake-gcs-server: + image: fsouza/fake-gcs-server + container_name: "fake-gcs-server" + hostname: fake-gcs-server + ports: + - "4443:4443" + volumes: + - ./integration_bucket_data:/data + command: ["-scheme", "http"] diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-avro/weather-snappy.avro b/airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-avro/weather-snappy.avro new file mode 100644 index 0000000000000000000000000000000000000000..5751adf0f5d5076a8521f93a08ea4d3dac169583 GIT binary patch literal 330 zcmeZI%3@>@Nh~YM*GtY%NloS{&PyyPs1yT>6enk-<|ck(s#YqgEJ#(dQYuPK&M!() z(oxDw%mwkoQxi)vQj4Gh#RZAUAkmW4;u4@xN`5ksK5S5lG*v=c6$laD5wnp=R7&CDxNs*SB> z$m!SIpK<)8&bsQ8g&LbrC2~z*OjD3BFf_EZG|;tVWR&>7c*2Wz6$UdA29d_!*VkhT6hys-vnVAaONNiu!Jw@#vqnRj!#O43Z2o3?-?>CGlzbIVq_{@u?Mwxdl0? z@rHWI#bu!(oD9rMYCfecyZ0%rw1S&~k>xuh0|VHo*x=}**9ts;--oRVI>3KM{g~H+ zqcd))yiEDey_4y<=ks$aRdSy0N+(a7$6QJ=auS^#ZIoiKJmd6w=gEqen*(j8=UB}? z8&cW6@z&$h({?K>{j)qI%hdW-QkgN_iuSD-gCPsQa8+9Zk~#q+4>1wJZr7j{s{iE$0wpB zLPs}Cwjw(CUfwmuyBDH`jon_Qg|jE>^XRELpJjS@JepV4NI`k|>Ea!8eZHtuUAn_{=XQOotxxs{PU=o7I9SOad)bq*X=4BTDGPLUcJLnN zsWdHQ{^{{iy`!0hQO0Y@`?e@EgNr9;b$h#LNwqE9#rt^es%dG=;XeY~jCG?osc2mL za93ze@Fznz-bIgo6es*Xb7?``m5GVnE{Bi%d*^d}o6J%xb?s&CC&xYdM_hxgzljJR zH;VoIK(({#B$wKggHa#k?zjF+{<=o|Q-boYTRRtb31!be{_;dt^{NM4Q-r!4)|*!C z+M4lSX3~a9;cwd0qOun){LUTran%yX8uneE7H_cV!j&&CeERSA zVfo+Hnz`#v%kfS%S+%nIi;sPtYrb;H-4|blSFFjnRQxwhpkMg3qujKxshW{V&+ly7 z_EyeczofF@<2U{Ql9QtmN^)E}njA-yBQ=vFBa<96uJTa=SZ*>f0E<0_C5<2!p3;&P mQd**wlmXtv8s85rl(88PP5roxXJrGqiwOw#0ckl_5Dx%jPL3b| literal 0 HcmV?d00001 diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-csv/test_folder_example_1.csv b/airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-csv/test_folder_example_1.csv new file mode 100644 index 000000000000..5f88160306ec --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-csv/test_folder_example_1.csv @@ -0,0 +1,2 @@ +TOK,UPDATE,DATE,SHOT,TIME,AUXHEAT,PHASE,STATE,PGASA,PGASZ,BGASA,BGASZ,BGASA2,BGASZ2,PIMPA,PIMPZ,PELLET,RGEO,RMAG,AMIN,SEPLIM,XPLIM,KAPPA,DELTA,INDENT,AREA,VOL,CONFIG,IGRADB,WALMAT,DIVMAT,LIMMAT,EVAP,BT,IP,VSURF,Q95,BEPMHD,BETMHD,BEPDIA,NEL,DNELDT,ZEFF,PRAD,POHM,ENBI,PINJ,BSOURCE,PINJ2,BSOURCE2,COCTR,PNBI,ECHFREQ,ECHMODE,ECHLOC,PECH,ICFREQ,ICSCHEME,ICANTEN,PICRH,LHFREQ,LHNPAR,PLH,IBWFREQ,PIBW,TE0,TI0,WFANI,WFICRH,MEFF,ISEQ,WTH,WTOT,DWTOT,PL,PLTH,TAUTOT,TAUTH +JET,20031201,20001006,53521,1.000E+01,NBIC,HSELM,TRANS,2.000E+00,1.000E+00,2,1,0,0,1.658E+01,8.152E+00,NONE,2.888E+00,3.047E+00,9.807E-01,2.924E-02,7.304E-02,1.572E+00,1.781E-01,0.000E+00,4.572E+00,8.161E+01,LSN,1,IN/C,BE,C/BE,BE,3.598E+00,2.000E+06,1.013E-01,6.001E+00,1.053E+00,9.252E-01,1.128E+00,3.106E+19,3.106E+19,6.612E+00,4.515E+06,5.122E+04,1.000E+05,1.466E+07,771706,0.000E+00,652114,1.000E+00,1.420E+07,-9.999E-09,NONE,NONE,0.000E+00,5.100E+07,HMIN,MONOPOLE,4.027E+06,3.700E+09,1.840E+00,2.000E+06,-9.999E-09,0.000E+00,9.295E+03,1.373E+04,6.913E-01,7.319E+05,2.000E+00,NONE,3.715E+06,5.381E+06,1.282E+06,1.297E+07,1.210E+07,4.445E-01,2.194E-01 diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-csv/test_folder_example_2.csv b/airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-csv/test_folder_example_2.csv new file mode 100644 index 000000000000..5f88160306ec --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-csv/test_folder_example_2.csv @@ -0,0 +1,2 @@ +TOK,UPDATE,DATE,SHOT,TIME,AUXHEAT,PHASE,STATE,PGASA,PGASZ,BGASA,BGASZ,BGASA2,BGASZ2,PIMPA,PIMPZ,PELLET,RGEO,RMAG,AMIN,SEPLIM,XPLIM,KAPPA,DELTA,INDENT,AREA,VOL,CONFIG,IGRADB,WALMAT,DIVMAT,LIMMAT,EVAP,BT,IP,VSURF,Q95,BEPMHD,BETMHD,BEPDIA,NEL,DNELDT,ZEFF,PRAD,POHM,ENBI,PINJ,BSOURCE,PINJ2,BSOURCE2,COCTR,PNBI,ECHFREQ,ECHMODE,ECHLOC,PECH,ICFREQ,ICSCHEME,ICANTEN,PICRH,LHFREQ,LHNPAR,PLH,IBWFREQ,PIBW,TE0,TI0,WFANI,WFICRH,MEFF,ISEQ,WTH,WTOT,DWTOT,PL,PLTH,TAUTOT,TAUTH +JET,20031201,20001006,53521,1.000E+01,NBIC,HSELM,TRANS,2.000E+00,1.000E+00,2,1,0,0,1.658E+01,8.152E+00,NONE,2.888E+00,3.047E+00,9.807E-01,2.924E-02,7.304E-02,1.572E+00,1.781E-01,0.000E+00,4.572E+00,8.161E+01,LSN,1,IN/C,BE,C/BE,BE,3.598E+00,2.000E+06,1.013E-01,6.001E+00,1.053E+00,9.252E-01,1.128E+00,3.106E+19,3.106E+19,6.612E+00,4.515E+06,5.122E+04,1.000E+05,1.466E+07,771706,0.000E+00,652114,1.000E+00,1.420E+07,-9.999E-09,NONE,NONE,0.000E+00,5.100E+07,HMIN,MONOPOLE,4.027E+06,3.700E+09,1.840E+00,2.000E+06,-9.999E-09,0.000E+00,9.295E+03,1.373E+04,6.913E-01,7.319E+05,2.000E+00,NONE,3.715E+06,5.381E+06,1.282E+06,1.297E+07,1.210E+07,4.445E-01,2.194E-01 diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-jsonl/test_folder_test_data_1.jsonl b/airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-jsonl/test_folder_test_data_1.jsonl new file mode 100644 index 000000000000..7f0b48d1d7fa --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-jsonl/test_folder_test_data_1.jsonl @@ -0,0 +1,25 @@ +{"id":"1","name":"4Gwxdvf30g98UC9LhQgH6E3f4oDnZcKXGBlZVFBtz28Skk3RrfJo4fMDWhteQBTV","valid":"False","code":"0","degrees":"23.2","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.509392"} +{"id":"2","name":"lIF1FmaftFbJmvzK6R5k9gIAbbai8VokXuLUy1e4it7iD883HNv2i7FxhHWeVgEp","valid":"False","code":"8","degrees":"39.6","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.509576"} +{"id":"3","name":"mPhAiAQE8HkdBYPye7G839EDWbo4h1OeYOeOkY15iWtCOamAUPUvBfad8hOKovpx","valid":"True","code":"-6","degrees":"-49.3","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.509728"} +{"id":"4","name":"ecBnZsegEpcSoqN8zv8CBwc2UX6x2OTFV9locGAPh4A8hdqprHuSP0uLeFDWDVeF","valid":"True","code":"-5","degrees":"0.6","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.509878"} +{"id":"5","name":"hIOCX9BMKnrvvLeW6lB7wr8IuMaDUTZLdiV29wxuucLDAMr6Odn3rYPlT0FvYuEW","valid":"False","code":"-8","degrees":"26.7","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.510041"} +{"id":"6","name":"jc81YPqBeTOipZtX317Vp4lmebhW89SDzUsFfFg9lLUJHHM9J4wBrY6xpkd9oSI9","valid":"False","code":"-8","degrees":"8.3","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.510245"} +{"id":"7","name":"yvnqkM7VjXRNcaS5LPtoA3eV2gS5lGJbYbPZANEAs9I27kxMeRsHrQhIHluuXtZ7","valid":"False","code":"12","degrees":"23.3","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.510434"} +{"id":"8","name":"b1biw83rVFnVVLxXFDoqhvx4TdLM4MPHl6Ry9kPVT0cwBn9H977IVeH1bNHJEW18","valid":"True","code":"-6","degrees":"0.2","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.510675"} +{"id":"9","name":"Rxpq81Wma1sj2xn9HbowSoNklgTAWy1hDVzPnWH8oYBp21FOnB7tInwVWgK85O1L","valid":"True","code":"2","degrees":"0.2","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.510885"} +{"id":"10","name":"l75qNDuZ371d6mEZUveDKsvsvaPlmSDYBnSPHvQu9w9wZ2shcBTNSET1Ne63MURF","valid":"False","code":"-9","degrees":"-28.5","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.511066"} +{"id":"11","name":"GFt2jG2ufrt41MxgMjPyQbDbVqrMupOcx2EnCVcI6N6MF1hu1qLUny8kwQ27E1DM","valid":"True","code":"-12","degrees":"-4.8","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.511366"} +{"id":"12","name":"yy89t2DdxqYxaRHaybEENQStJhaKhj7GfY9RtQWNxi48ywKgfnfUGaKFXXL1Olrk","valid":"False","code":"3","degrees":"-11.6","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.511672"} +{"id":"13","name":"LqvLA7uXVIMyVkpNBkzgNs7Ib8BuaAhQ7ssQ90mzC0N1Koi1PJ2Qf8Lu6LfFz1Rm","valid":"False","code":"-15","degrees":"32.8","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.511892"} +{"id":"14","name":"XLyPTcNivK24PrA3WPE52JFpSZL4c6YpBY8WFggDrDlKobz4fNAVfDn7FsiRCQ3F","valid":"True","code":"0","degrees":"43.4","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.512060"} +{"id":"15","name":"r9hU371sBIUHKE6RT7PhjvXAA7ZIIeyQCxb9BBVGV66YvXySgrKVL5ztLQKODfmj","valid":"True","code":"-11","degrees":"-5.7","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.512219"} +{"id":"16","name":"SGrCtH4FzqyyLI4t4xXmloVBzOlv8YGr3I4c3nrbnpze5lqu6vN250a5LulgclyD","valid":"True","code":"-15","degrees":"15.6","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.512372"} +{"id":"17","name":"XYY7qO6LWuLgf2zUcx37qjE0c2xN5OCHoH2n7ifzEcz5m0b1nygFWVbdJOuQG5ST","valid":"True","code":"12","degrees":"3.4","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.512643"} +{"id":"18","name":"jBCS2mcUQAdjnGgnVxXcSshxXI7NwitVIxXzt6yfRrEZ5hphGNqPsWyRZywWISEK","valid":"True","code":"-7","degrees":"-3.5","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.513031"} +{"id":"19","name":"UnvrDcleOv2F1JkhkMrdLoiEH6qgyC4Xe5U0EbJ27rEPLYPa45Z1wDMZXDMrxAG1","valid":"True","code":"2","degrees":"15.6","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.513273"} +{"id":"20","name":"jgMCpNlKvxmDVhxlkte1K8lm7xdhYeVmjg45nNksCl9OzmjCCpcXeWTrsxbjqUmC","valid":"True","code":"-3","degrees":"-43.5","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.513446"} +{"id":"21","name":"49sy2s0HkHvjHs35L0kmKgRbeQdPgLMeXRGCgt05rupXo6LPZzzOyCCacR9nItOA","valid":"False","code":"1","degrees":"11.2","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.513597"} +{"id":"22","name":"QnpRkd4kHPB7uGbWkAQFMFwiZbBe3FWPtubgzX01kfGrfHYPU2GJRLDfddB44nkR","valid":"True","code":"-12","degrees":"-8.8","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.513745"} +{"id":"23","name":"IchQUFxZYGMiYbhJ9NcoKMj9dBVZhwIJhj1GVgNn7zPKZF44IIBaf1yL7Wgc4kLw","valid":"True","code":"10","degrees":"33.3","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.513897"} +{"id":"24","name":"Uw0NezDGSQ8uH7sIA49xrRlnWUihWjYCNEmdmzHcPlFTjs3L3L3Q0x2486PUj1Pb","valid":"False","code":"-14","degrees":"28.6","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.514041"} +{"id":"25","name":"DlPD3GBj3Xo2et0PjhLtEklo97UdY99R0Ec7P9fZ6likAOIllSN09nbfnGqKbf8Z","valid":"False","code":"-3","degrees":"-10.5","birthday":"2021-07-14","last_seen":"2021-07-14 15:28:43.514182"} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-jsonl/test_folder_test_data_1.jsonl.gz b/airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-jsonl/test_folder_test_data_1.jsonl.gz new file mode 100644 index 0000000000000000000000000000000000000000..39b59918526cdf96eeb37b0348dc4b8c8c71738a GIT binary patch literal 1800 zcmV+j2lx0NiwFp$eDY=h19W9`bYEs~Y-D9}Uvy=2bYEm)bYWjHE^2dcZfpRpm&uN+ zN)kZ#{S~cRca_K_ncfb}XamLr#cNEpH$#rScFFc_RM& zxP~8pe1MOCeN>k+dOjlQF@{5oV+li>8%9kJiA)N3jB38}f}JCAg=dLeH$(cA=lHRS zw>642-cGydz@4q(chNC zyd#f9&Lj%!`i?WikH)3~e;@JfkeD%+8j2)FME=r;nK`6Aa55ES_$ zz+VKVk+N6zAkcCia;`B<klYBljl*>(xl)ZQhE+?-7VM1Z-^#XlemKyZ};Vzi}8&+f~glsF6(;9Q9~u zrPnUJdG^%sYC+~P_I;r8tk#hGu)-VHD9)HTxV>P1pGBYVX#~kHY7GLwFa%x*G;`u-vlU}8VG7mU%Cg36(SvJTa=hgsQ=b&(w-M-d zSp+bMh!+9%BAABio@~AhpL7&LoUeU#N>u` z#)1b|F)~9@iWSMg*X6BX1NwJs8C!Rq0*s^2QgU|ccz0vyHJ{X};ZLB~cby5>S+Jn1lRui2+4v-Q6 zuYI3Jzf{5}`l8n{;BO2IF;5fCbdV3nyE|Tv@@4V`K|h?jZF#C&Vha% zb`Tj2i`#}kz^!;Yc`6D>_r|%jEhY)y@Wrn7XB6wK74sD(w%6n4fPXLN_ZIJY3bcv< zVkEX8SlGE?QuA5QEvK0BP9TqG%1E=pVOh#klxR9^MxWt0OR`AP9h~WuG&@Tr=1x8A z_IzAsKa>M_&n04|ogoCwf?=WOlhkZF8Hw|Knp6dKXmrY=wzzp;474jXxPk(_YW(Up zM?`V&$w7w%b|KWhNP=SGe_(jeBtWbz2Z+Cw!$Q!G>r(eUsoK4o#Ber)5vT5%fCX^W ziTy^dWmr+ycou@0C@~*YQzF`yA8z;kK_X7)ZyA2Ml~!jGiSPx%!p|AEIl(e84{VrK zQc_u?6P(&~bQDT|Tsun{ow*~Y_}B;@k={~CzZ}>#HD>_hi~2Cn8Kt@VAg^1VeR=3nD!5;5EM+c6JOE*l)xOTL}6 z2T>CrSI_4nT(pg@_mkPdKpsm?EW3sYrR`DW<1pk9s`BISHyQLwx8c-sVt@-lD?xfN z#nE#m?dtmJmeGTHr)5mYSzh{3wrK_=E72=zHg=wfkfLyx7)&ZDqqpLz`RKK9TnOB%h+o^WAkzA$)P;xgQ+Vbozpma3hQ7d1nfq02BwOb950i9NEo q=ImZ6ij$5pl^<8qedout_XEVQ-5$2GrV*f@dixLYpNSX?6953mSZXH# literal 0 HcmV?d00001 diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-parquet/weather.parquet b/airbyte-integrations/connectors/source-gcs/integration_tests/integration_bucket_data/sample-bucket-parquet/weather.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a31a6f500842047c88b8f8686622b84f5c348aed GIT binary patch literal 21006 zcmeI4e|#Kuo$$Z2nc2)FJLycDO=n9QH>C-iw%g7$O}i;2-B!wNHQ{zaWy@(@5b|u5 z$SyzbR?fqH5H%=0Hi(!iax0>@a-fMt4IZb_Q=a84KIw@Xd{A?UHF%1tAO??zd*0tk zTUzn%b+6~|%d4|9^PTVa^Zoq(e7?V$-gS`|7U5^(;d1MS@RJ(P!EjKB-?g2gns^=4L_6Lz#eQF}MhXd{dzZv|se(#3r7*fZSnPb`5YAt<&nUUK$O9 zoac&eGe&J)Y%ojqIKT96M(ss!mX0?W331q-w{v2HQMDJsy70`3h?`NcZeTHo ziEMxQHDiGed+Hg-UJ#Gc;32NXxYW0FW==d0c*U%!PkhZti{;?-nsC)9sM@qKp?-I# zT}4z4rmjA>(*Qa3yI(UV?6g=B7vhEOB>}9M>>(+BYUb=jW(tnw!M2h~&{YCTRwLU^ zvrwGn$>Mh*o)!YthB9y7%Lyx<;P(mYI^vh+jCcmfQWBB)6fD!lqFEEq#mB`QdJ%@5 ztN8X}+*0-Fo)Wz$#82Z@p~K%sK0Z&Sm!v#Yb-HJUHdUcf<=-g&C9q!NdLB)0Fh|94 zL=}Kq1zco53)B)-kyRHaxq$YkDuRvT>Gy&@&5R#Ia zGb>|xG=`G+_}J7~VQd;k)rwMXE)esOGtoCsKWpOdJUv{ z4An{@1U=*MN6)Tbzfof_TBqPnM6@Y!^k39Z!kxULVa&qe9EgSVz5yTgfhW zR1G}PD59u|)^O}Vx>!G6#pIj85}2^F1m;pP(|$cv_vqZ(9(3zGed3N2 zKJl?5{0%0O!-}FF+ivKg;cZF$WOUO~`-p2ep<}M^1~=28XXtvlH>su6BU^dxH!OAV zTr@@(1H4Kl4<*BvcC{)74C$2|aX6wYf1?i%_Zl&^yxBK)q-zFR32JHj!e zZqBR2Tkt5ZNEohK-i}i|(r4ft_f zLToWgs^~ML@se6TkA5X+AnGC!PIeXSX>r)Bse?PHerPAIx4aAIIh`~dq-ya7GTRVU zzr^xfs0!^e^6GO*Ol~kf9p*;4}Y z1U6R{VqfCQWzw(e@v7R}Z;Y#v?Qo=SI~m8Ut49(>3=FCwV$Xwnd0%-?T^x!-q-R9M z@Q5Xj3}wB97_?{1tesGg>;&$4_|tS>KAsi(hIDawWJ0|)X}Eo}@x|hV=-n6VaWdmx zf#5JLdd*ZkPcSGG2olArmo4V-`3ZZ$){7-NB%Fj}UDIONBx>~y&5xuB3t78jkD95z zsiARfxI)~i+77;^*T9cF5K|9thNFkJ7YN8hi=vm{K?GW)TFOqym_Yo{J%0C2Niq;l zJfR+TjX5DEE5e1YLw3wu5F%3QiBh{4+9291JTQ;1N{bktvUh`#3uiMGTel}5WE>SJ zBi%CWj(Bx*+Kz(R#AFWVFslx4>JjRZ4Vc%)t-9#b#inrSeYzNYtga5PM~gPbg^6CR zGiKC7=kOjkbf}ElX#(ITw0QU&BPWg!(l?-sJ&14}0_h3o4%8{p;|_5ZmW>nL5`0R@ zC`WEklGQyOf-M447OyNlb5@1{$OGRYgptYHQ}ohB`?BZ`TLD8anNFUAhmFq(OBc+4G}d92=1Grl+l8@8CW7p*<%FXqgtF5o)8lE&l| zG#{IsK3-mxGsK~>IoJ~U)r2_QQ8zl~3~{bGZs)|tp~@3t`9zZ_AD>xep}fBmYbs6g zTaKhKUte@%YPUU~5qRKQ6ABDs6J>F~aXT!myd5$h*E~q}blbl8oGV611 zii!Q?j63ySBCalex;rQEF;DC?s%n4Iwg%F|a4x9wBg=HL6>`Zi0HOREf43?;{U${S zYj;%apAZ8IZACbq7BB3c5=IK|UBUC@;Vc;-^onq7ASZepv0=9(URK0c6mfZ3p!h4o zQDXwbn?n$b;+fr!@?&7-$TzY=372feh!RZ+Lx^hut;+A4p|c*D^+=1#Ur-9%s0e%X z&T`-a8Ya<%Q6LEY@}w~4ADI<*D`KxBemXKPB3!+fR|NHYETKM?6qgbp31%;6#Xwes zm4Uos)J6MK*}%(+Pg14ml15A%n`qd(AX;{3l>uGZPcJd?%t%7C+%ryn)k*Fxi4}c0 z!@8gp1h$bNVy7QTiW?LWuHFz6eU>qI!IawLl1@ww&omgZ3!>mAo(toD-(PEh(>;q9 zh{mm;y%3-MOr_DNi0ArZM(Oy1cp~h84FBME$EYq=@2&?x^3fZn#S>Gqx3oBWC=2Ae zXx?4&X{&dS>okCMS?r_o#JJ&Ldd_Ej)YjcoZRm;i7YyszygC4e{*dBf>yN7xiV|w} zhY_ls)RtMjEB%3l-Yy3iqHVi_288+H#qzU{1b1DuZ%6wb-|A)8dL(9yl9fphV96W} zY3igp6`AX~woFT-WDW^rn!*xPo*MAFFYeTSq7KXQ7PKkKo$(7v@gj8{GM80IUb>j>?KsR19(qXL$H6kk zr@D{f+OXrffL#OT>!D^1jqzW0Ri-m^Q38MqNVGmaN+unngO#9|fFLVUvque3$;50d zgSsoj`kpVV&{6g(nJm653=nHOR8KL4Ft(=zASylz$xJm8Jx~DeSz&~s8T)HBN6C+( zJ)@!nlZj;}GBN7Ilij*7T);|pLy33GCXXzT?d?7_f}#3u1zFPUj<{ ziMJW*e{Jli!plwaoA}i;ic{I%nXdO;lC1gu~-pyZmIF?*jX$~ z4_Eko5ge6p0^n*`x08lGIf0%CoQEAV3j4>g6gvGWDf*)bWOQ;?ybvZd7*pW>C|Wcw zjLRALBP109b{)ek8UPY9olLSA-DknY=kPcM{LwStBel16 zqZ+9jL9-XmYwOEgop=H#rbu8-p7VW(4_S8gkSSN?9M;7rSEmWJ&Oms(e zBWBm%wP+L~(}qPtotq>{>JSEjIzO34*-{+~`tj)GDBow%KFs21bYNWb5=ZC6T6ml~ zUYe}GST~{_Q~DTVX1jD;@30J{ndq1?>?Cbg#!Fbif*+Gt4^<*lJ+sGWJF*5rH{X$J zTHA`8)y@{1jiT94Uh|^)chp=BoHv=f%!fo@ zcs+4yZ`W(}KI+}yb&J>*Uk8S3A#%00Ir3iPbLO+ui0crcodfLS{(Q;ip2Ow`?0@CU zp7?JGqj!tmp0e@F?UM?m?UiKpead{J@Zzg&J+6rUp8F-26mRd@_SyDNh1aveTu*}e zHnBBxUDw%=knH*ht$LNgu5U9wcq`WyNfP0!s<(nQkPi)*inPmIG>v^9A@z0MOxc+O}Uy3a~|5h9r`sALq z<`qb0&Cpu0^_FvhRSrM&z8gN>^v3C~qUay{xHcjZdvDT0;Z>h*7yb4otv!BexW4gh zaWE#f*z2_G4_+K!Z{B(EM)S|aR(sv2RrvG<(bwZ~|K8ByaEsWw_rscZ@L_ncweQ%$ z)rfAJvZCu{P22NP^}=0Bcu(x$Rpuq&a4pwy^V*Sj#NUMKUUcQ}wcT-pvfJaNeYXs1 zcgODpmz(2X-SacLdd7UL>(2NQ7`~}%xB1aG?)_E*Ejxz5x0>%auM%#LW4<1G&ZZ;` zwO5Hg`}#w_eQYd#hj~%_;K=`ths@WHydJfFz4kynaQ&e@eemz>zVppryzhDKJMn9W zR+{%daNf|*9(!u&F4Seq`z|luzvlzyyG_r$KzO&bYoB=69hwV%XJHnvgOpttnb(^0 zBS#;ykA6~P^wX2s`WJz&=ipJADz6;_;-}#qP8}} zICt`P;q7e%ht>4|z@^4(#paFQ(EGZLOCGyRZ2jUd9u2^gk?#G*mAf&co6*T<4>pLt zy;sM!ZCvshn>TI^tKr9$@ZHK_cug3af4dvt_rgo0x7l;y9{kM37w=J8b$*mqMH3e* zTI+@K<0a-g<7_dmsX|o(p@2dA zW}X^U*_sxVGy|iVCt&E!v@!xsK|Nro3Uv(HhO{>bQ@cv&%I=UX(sFuJpvBxy!4=vy zQ9WP=meD+*y;+z_Oazr%RU@F>qnoPG-rj10tEMVna@0Tzz?+qNjOvXvFu_8pDXKCr z6y-s!yul0zwNY*6V^CFotn2jrY`T$p8dzu)>rkTuq(Lipwu94C%39?mT?85xU3s=W zV5!Z#nA)7H!yc2W%`nF52`b+|U)7*Rg)(aMye!bvwGjxHqKV)#s8p3pMX*aKZ%|xI z1cJe4#bqeTH%;vfh(@iZS^?z|$E3tmHL7nAs-4z@fq;@!143(8+Bz>1s&<$1R08&b zuHZKSB%$5Zxn21nFuHZb)fHh)W@LORutA-Qw{zp#-t=kl3n()V|H~ z1w-o7QerbzHyi5^=VnT_@wQc4uJ&zHw({U|UC4*4)I_i50LG)vX6=L0=QzCV;OdZ# z)NKjH*|K$;#HK$Xe-kcB`*c(IKI3$y{ereIa6)S#S1Je$ky;n@B`GzrkxR<8iC?LK zA%xa~Y+87d8ltqWl^P+HoR{T#P_;B&2|9+axwe*yE$4%VOjZPzp`nJhW*ix#2nxSJ zptMSr5yJ!Gkn}rl1zo zgGOt#5eO@(wo*$e<1rm&fQ3l}R`H~0SGWfC z44!q$L)t*6W7%-P)DlXEa%pGQ#Q^AIm3kJxLc6PKfm$P%!TVY2DiE^y)(+%Vfz^St z_-63D3^ML=gT9Rffqqt?L%UgdUOAFz0t<_pM&LDpXrQg@O`#5~K>5v<6|m(jiW@;_ zRiN{mop)Fb;Mron-&)3}_O9)r74T4b%9U(uL~o^2F|{M2xvO31o#}R{#f}5EvRSB| z8Y*m8u_osG+CWG(rJPfG(4>NfI%&uU&k8p!sX~gr zAq2^)Zo;f(K_#(PstYa{=c%+Z9=L(;0m%eadO{^7J&D#z^`K@61It%6%~BG$e@nS} z3m6~*DJ|9HOPex~mL_I`uoRcYvor`zJ3k$u7wLE!QCbs!8RA>Bb`g?$&}yc)pz?+A zsIr4)JOeRf5}1z5B7rp1kokTcd(nnN(4Zd~6OhDLYgGRZW>oX=)G&1whY_g~%oJharE1R6Ep6DoumZtomVp4-tQb z5i(Zj#B^-~U@xSKX>_Av&Du_JiShchub^n2R3_2{AB+t7^G5iqXDHnB>fj$JoQx_& z(m6tUPzH196Qz^X(K`_J{iKv;69`N9MVLhGaCt}(^$Ro_10lj;$Y7#L*^cToXcs`K zk~5ZLd26qb$-+`%fnw5E6M}0*&iKC@RWb#~kx>n)Yb~NltF+=o=}q|%UN>(s!-pz< zvlX67{cW+Nn(Er)mUQ>J_M~#Yjsk0=qS@H4eMb2l5&yb&kP8A?V=e<_c8STzmWY64 zmnpv?t*4%!7V3}9t-I~w5?rN)fVL@?KjG1IwHPk_ba+9BMM3c zMx-g)Z%9yYzYEUQ_>8h@hfeeaf#RArwmG5^B%AyCA z7UfIi2r}i8b|(Xm^bwjW6>2mPK7t?HD1-aWSQs}s-6pn3rRa5u+nYht>nDnPp0ZkO zBbVHawf8E`q|kke&D%;D@k81k)!RljB6W}UI(73_#a4TJwV=`C`tiB9-v`-;p54(D zzH4c&_C#v=LG9gV=-HX6r5~a-y)}s9kf^o5OX16?L0Q!fT{0olw6>TGfHJ{C_l)f! zWE#}ofPxrONiSD6J3%ywj7-Ze>4WlSz`c%81y^CUOw3xec7m=Eq?dMOor+@Jp)|=n zQ!C(INFYm!6O-z`%vViAYr=67)3vWC9b%a>K@MiTU#o1839a-}_?94fON;h8(lLC0whD8kH zjIH=QOSyw^H;Cs{p0;FUZqjZf(`^X^_0BxX9K**wVl@-DR&-qy0|+k6AoMk8Q6*1~ zYH17jIU75`6q7=xDZbBa#jd64q+(byU>np{xb__sQb&XEau4I@UZ=|t{N~R03thXz zSQV8dH;~4x=#0YFC!m)<$P>jI%*B|(y0EzqXbT-l)v zSJL=bpdoTMN9gp4l;5>QPiROhkw(}CqYY*>nZ0CIm6WUEa@Xfo{*TXp^R`v!7+*{6Ixy zfWlP7er65FL5sPzp%U;M(3odJSoV~*U3N@>3(G#yF{V11F6=TZ%r@)LCChbB$eOa# zI$&7olaDp82F7x$M~W-QUJw_0boQ3Wo@xWkY>t95n@TKLsI&eD6~!2ZlJXfZ%_o{V zfR+_vesK(vtG1pQ2XxMJY$cOpk~ht_InTAr0~U)?^cAI1R<>vE&s(r#nYuR(0y7M& z9`zYhvh|~LnJm1D+DVVyC_o(ry)xuZ3=|Z*0&uHX<8>73)4&8-kYN4Bi$RG#z=WwK z&WR_1D+L|%N-Q@Z57!d|HQ_Tb?MeL0l5hYxfRYuM#)iNe*R0qQ7hXQY7v__CnC6zk zEWpJonAdo6ePLL3-jghn!p>!C^c4f=1g+-yO(7_Y_f%%WtI-bHn5B~ZVj-~oY}~N? zTe68SnSb0S&X+ckmNsz_nm-^lTwH*z@8oS4Zw?cguVa#aE6*wMTE4`2N{AFs=Tc_} z*Ljp!B1_4i@st;DG%YmDPKrI4&&3qg#BMA@vY6QeCeLePkatIH=eH_LAzv47F70ZBj#knuQxtk(A5+aFV@6 zC}=FBqd9Y#5{kj%D4bnn?e$MgDf5(Pc@_BV!pSl^G4IWb*R$$5%UY){-pb;|ggxs? zxlDsrG(#=uWOqR^$#QB*YH-eD&Dc&4lmvP>E&kLj49tm)>s5;-)-*PmiA>JRBf2S2 zlWOaTeJmjC2X6;))y1VCJMTpiXA}!X-=K9K)GA}xenyC#SsF;9Dd>Up9eF$I<`x6bCgN-Pl3i1JW6;@MtXx9Mwx&c(abEOn3;4l^O?FGg&WbaBqWU51&{Y) zZLB($!c0nVDmyklR+@B*bIgsg{@{t?;-m|a5^GojOnodrmOPsG@O_yC{2!mJPEG)a zO#)JU9_#X~hpt0sdf8f2^+vrKE;!06##;2Lhb;v8qh&DGy#gW!?!b%!(q;UEbSjIG zpcm8L3=KtxYYZe6Z?wKTW2(%kldOM=Ve>gU>o zFfD&}TW%$kt9k6sW9hCH{Nw^)8`ny%Rb1@9Vu=DB^3Cv1c`6f}EU3XcC{6P#bxUq= zTQX7B*nE@5j4@~3=Y$M^ayNhrPo&F`8|~f%Px9E6HIT=r%Pl&H#AOxU@p4$!lIh6} zY|CMp^fxVhiQyA0)`!;cJB2;Pyejt7jybdT=1tjkl%<4!NMLt5Mz6D2#72fpGLu6% zx;>wfE1&w;mAf9rjT9krMVi(z><&|+O(nm_C z9R*H`UgZYClpm~I>QrIwGGaENfhF&9ri{{+GWx(oK%CDq>svq!kCG_^k~rrP*fRwP zWLXUzVtFlt5@eEyzsmPGX}rL5pytf&vvaeB}MU&h=_3Q&5cj)|}UhKf$x zv9UC(YHOf`&4@>xObOje52OeU?7r~IY^g&9yzyd!?K!Vy2ZKJD_VTEt4%Qb)`o(@+ zon0v|*b`%QTo=NbTtw>Unxqt7lxvEvvfijYU$g+9qOr7z5`Lt799zUSJNROj_izft zw_?rCjm;6)FJXaiiKXeWnm3E$C+IrC_RNd6=LF_RoW-;%u%-(C6W)YZ$mEHbF?Fm^ zbTab@ecqlLlUqloClMtLU;|!Tk_*QO5)Rx$+>;AlVQl_5G#thLB$E*IyeB5-@ia3q zDv52&(5XZ48gr&YZ(?27I+`rjGL^A1qABn(hjlJ^Q(kh+Lhhp;t`+iUj=Odqu+x4> z!8aa_wVP)c<}dzWr|-#KZT{Z3AKjMhRg}=DZc8%06Star=&IY~J~#2tsK?8uzBW){ zv7SG@VxVTas3bMHg3101RE=GT>ObAa=8B^%$Nz6y9A@j*Ki+0!MIF_DoLZma{t@mI z7}C#(anEu85%+oSPjJn0J<2sjz0U*SJ!Vzh5Q!?kQ-{A#@q<+T62*720Qw1@XV_FW z&-1UPt%!T6QQ_+muBRhOtsuUNV*f4O$z8~|vRwWd+Q{|Y7b*KE${u0hT4%ZT+gwlZ z?Wm^j3h7H7M>OV0;_+=H+2Ljhk|~R_q(Zc6FvPU_xG^IOArwE%Pp=C)9o=T8zZAa z8>zuAR}EGU%!_{q;xDNkptlBEHkcDEEJrEJil5W#^YHa~z9|5Z*Kb+>UlgD7x#ftL z;_MHh_4l~O_)3F}UK7jWj_=XsAY{=@ zeIohEm1^&I>Gv3YhRk`)G{l?SG1|uVt^&A48I#DJtV!{I&1vxy8vYHk--n)E!}T=p z!px2(;BpKo39z-qk7Xzy%Zh&`yTlZsB1fs|onA2eL)ke-IgdD?X8#K3)^g0*M{D$LkoBb#&n$5?4CnFU*`LH^42BF*%tU(sMDr z8RVPm$my6sROGon4=bNRd~Rl;BgHlXI48GVWC%yYO zXp2uHA6@)w+{G#q#aY45@XW*{o44atu`)hS_$ZCdpd;C$?xhLCCF-ZZt$K7Gc`d@W z9Qsxl-)G9> z=OOI*$3)aRlKuwz^RU!9(_$+_$DEEsiLzzFVH~T2VqW&CT-Q)pvUnQF z-i0haPOY1{K1}(ysrMuRe#p0b(Xlb=-^}$0-~O344|AX3eiL`OJ(Ct% zzJR9w9rsDvOp){C=pl+9OdyXKNGI`14#}7UHV!p)@io5R$8{agS;{Wteh>FiWLW_6 z6yLt;`}7f;}3$-XJ~&H*Nt2oc$dQ^ zspNkS{EzegiHNhOD#n2%d-)a+=6v}}J|t-Ef`x-ma2O9-;vLZXEmwQlgCr)unEJhMnN>&yK2C@t`u?q{o@-z^?5A^RLx)1UEL-2o= z>xW?R_h!8~PoH`CFFp6i=-`8k)-szuJ5ydPrjGMz3e3cL%FK#8cKMhm`ehoIpTVn#KE-8T`GWdqZl)Op(4J` zKD^aaaO!@@_yL7a0dtNDzl=CvZ`?JZB>-6Q{r>~7;%av4*8o}rK$Pf|Rml+2q9M+1 zIT|lHCkq0Ywu+~5A2`{^a=r_ZQ%oV!Y`J?cj{(X&qDc;9uvhp8q^qfiez@!Yy}rmfw?pOj>Xpu#Yj894lbW3)vSY+BeO3? z;~&KjPV2|nUkpcNTC~9RFqD4+q(4fvDax;7KizkDZlV8Xh<}=I|46;txIPbVcw-ba zS(F%-#Am5_KXQ7zsdtnZYfloLhsvaXIX|12hEw07;g29p3j1fK{tuP+E#ea%H%B4u z1v>aH)E9KzMV3S^dNfI4F81{kuJV?X?)BlOGu`XSIm|{mfYjswk}YpUbDcqej;I~j zCKG8H=N&eo?sw#rrkrbIWJW5@kb7W4j%#D$tFB<8Ai-csrxqhNoebI{o&&0GTTB=N zXOwn|oK=h23d!~;-x7YN?($KwqXSd2R*pgj8L~Z>zUyxlg*#YK#o|}aV#zr+N*L}_ z!!ME8P)knJMHx~=J$FCzmkN^y`rQep?&OE0>|_Z8mob6DNMDB_pfQ~)JUW$=ZYepn zD!bGn+3z<`j}pNN3ZoJbgHdKu7+~5lIZG;WW7q`blmsTHL>l=-oIs2M26f)F zobGe^Ch4YrrXZQ(^0h#xC5fnfVZ2D?37#B@4Pz%T(9F1-25MFtg;NnkZsx4ygI~yEU0K~VcZ=R479?sfR#-O!f6IFsuGru7938) zB}N$(>dJYtQ1YZ;=g>YWh7_NoaK0pw-;c&mOVP-AZe7mA%9e8UEN4s|W+VqJsZ&zq zmd`Q7b2lvDw8j3spxJ^XZd5S0Clz9zOY&8YH4}at+1A%uuw;ZPSzg``fC(C5K1|e6 zd6e>mZ$p@xSCg1gDj7QOKacY)(O6)#8w1b!5}sKaqGHU>`5Gmm$X0p3B@-IZOjdi>;yfQ;Lm~&A^`wU zl_nOGwk^RjVE{)whFL(CS_-XmlqQzkN{o4sv_0$hEvF`B6KU>CER!=zOVjrfaGH*y zeodc=erILW5a5flCn-&5i5CU9&1A7$yila7Nz;wOfGi4#IiZ&nO3^Bwgq-qw9K~-z zGa5Sz@Ao5&i6tLmdH!@Kq%mDkhX=knNNOdSo_xqNp^=pc$eG|=yIjMRe@j~Zxj#4s zchV2bQz7~WIZvRMJq{gk6xRxg_5QIpnI$r@#>RHK4Q=o@`!nOlSs;RxOK9Mnl6shl zNOm*H{o$6{yM;+U@FsT8>&kmLW#WN{OV{%Bl88hct>7Pm zsTiJ2zl_{EVJow861mV=w+@iwNg*mKB<8xT82h}{;?au`E|1usR%*O4=@) zFO%xw1nICm(<6|0-mXA##G8mjd!lmY)t~K*a~7pHUKPSia_Ws9Y;VycH&@xff3Y!T!OP#vkc5;#F}`4!LcQ}jhx!cF>zk9j9eW) zlo;8{bGu8^&C^VfSCFn8HmAMxp~YL~#}+}4W8j%+MrX|3MeQCCk!Qr1StZxWGRa$I zy04utatfG3#@+%wBp7zfv`!~!jM)oP?-(h{&F2>V#W*mLpru32k~~IX*BHl?=u6Hr zv>>1;@K%yxPx-CnArnr7Qd_c*83-=NKhdz9(f2&_Hs_FXuH`zLT8u0uY(~jz!DiH+ zVRXC-tk+P^i3ilkVy2g<#JzwZ+Q7jkJ9AkMv`AO;gy(6nN@S8`G&%4n`+b>Gae$am zU)uR-x<9*jT772cE$qm`d{$v2%fMMP9rgL0w@jln66uC`#V1)Fs0dNTb-AI1I5Ek|f5P$`pXsBS+kN zCPg}U7W33;WIBx;^nK-?RB`5A6Ft?jQM)|0K%$Um)^t=v!dWAgDP(lII0sEHKQJo} zGf%LIk$iicBRtp_JvpngMn_iX*vzcKH(YC`+7WA-oVIh_b-gFe2&|hk2~)K$oaoWt?sh`g&ly;Jl=N!;xwwrC`XOevTR0&u6j(4oO^Mz!{_3p2jh9Qdlx<+ z5>Ik27C|22^B}@M#*xh<+xa5@BZNULV8P2nOi4}3^TI3;WI5zq;KXSe&r#mS(PR!1 z#tZ79&9JA>Obn&e-mN`VahR~o9{Ri(9Ez!j2Z$)4p%_A+^@sW#$t6}z4;3&9htr-@ z^1yQ%v__dVt;mzgoT{A_z1y7t42&}}f%VfPx+G|hHVF~gm)e7?PxQbRjw|EmaH)~B z5sS~kjwFtMB3yPF(gAy|)fc}CWQTNzQ5;Vy;`Cl7(e<04YDimb{H&2V0p4~GucM7hZS zF;L7B9bz0N?PU?AObo%vjpXp<b_7?z88o8r}j4irmPaFQ1?4#5`A9;=63 z*nX%pE|<4ba&cH5EyJ4flZ5AXBanOQ!*0?JUA1na6M3y3)ExEv2I-J~-*AIr$T&J}$l>oKG#i{3n)R%E@R{4L8BHk{r$@$66q z;vFO@rAcmzcR6CGU$mZ)WhGEdV#)FpTpq_N^EbKp#R`ZQeV^mM$*<%Ff!zExNiAj%9}c>kHr8nIHKbPb8Di8v^Y^7n`Xr|T`V6hV|S9LOf>Od zE*#omrbZSf#|gCAYW#Nq$W+dY^^WC4Jf1UCUo6{|2gb$V1Jk0n zyFjF$oPVGuj{Rx|vBn;&iNRmR5Wj_z4w8pgkJpapDe1NA@-%S&(lq6v9Z9yetqOMx za-&Dn^|v=(Eq4#?yYgD{!IuKz`vU$b=fv)_uPUh%dy4`AhA0fk*`OORb%%XWJhLm1 zomdRjHOrDa&^k;5vC3(^&bZjEPRz=?; z&`nN_NQ*#QeT(4Tw+JlcltoZmWuq(>A$*Gv%Xwc#O)NsWBG$y4#ak=8eP3k{d~>il z?13lQTZn6|5)MROTG>Y;cg6K@d&d=*zxCR;<*yvMEWAO4o4Nu{7hQJXox_)1b=A_x<~@IYS>ftyF1zl))$bJH5RLVVuYc#Y`2+8~vWxMeENH&u zz&o$_lk2a&?)(GSoGdd>mR(%9^2#etmNv;!YV}_>@`pveg;PZ#S<7#CqMk}8uN2Cf zYmt!bId%QWfh!JN_byrS56e!}lk_&HuetKtYp=iNO6r{;>tqSFmSE|xz3S@guQ(0R zQo(6Z8YPoXLGa)F){+N?w&oR^yp3r@ztd#AclP2ol?mqaF55@p0vAf-@U^LT)4|o zTp%nxoNCl1tjHO9yVUU6=CILe+j(YEH~^O^0>)bS+g4m85Kgm)rRIMfYJ1k`uDqlB*n$570ws@~Y4CS6ZLyUMg*< z``!;I$Z-pm<=>f_@#<2lNX=M+dAdC>Aq2=Do83n~EJ41_2Pi?VJ@F(}+MuP!}4>QYLij6)IMIlSV~mmtnIw0*Fl`y1nd?t8}r7#GPgR&h1fp`Lm< z0pSFFC;a~@gzh%FKlgEokT1kzr@P1h)M=hT=!^)m1oZKSwwoHde{pZ1`^dfj2~KV0 a*&W1`Zia@z^GOHJ-z{DPSJM1XkpB;D ConfiguredAirbyteCatalog: + return SourceGCS.read_catalog(f"{os.path.dirname(__file__)}/configured_catalogs/configured_catalog_{file_format}.json") + + +@pytest.mark.parametrize( + "file_format, expected_num_of_records", + [ + ("csv", 4), + ("jsonl", 50), + ("parquet", 366), + ("avro", 5), + ], +) +def test_read_files(file_format: str, expected_num_of_records: int): + """Read records from bucket and assert number of records""" + config = load_config(f"config_integration_{file_format}.json") + configured_catalog = get_configured_catalog(file_format) + source = SourceGCS( + SourceGCSStreamReader(), + spec_class=Config, + catalog=configured_catalog, + config=config, + state=None, + cursor_cls=DefaultFileBasedCursor, + ) + output = read(source=source, config=config, catalog=configured_catalog) + assert sum(x.state.sourceStats.recordCount for x in output.state_messages) == expected_num_of_records diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/utils.py b/airbyte-integrations/connectors/source-gcs/integration_tests/utils.py new file mode 100644 index 000000000000..77871d561546 --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/utils.py @@ -0,0 +1,24 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +import json +import os +import re +from typing import Any, Mapping, Union + + +def get_docker_ip() -> Union[str, Any]: + # When talking to the Docker daemon via a UNIX socket, route all TCP + # traffic to docker containers via the TCP loopback interface. + docker_host = os.environ.get("DOCKER_HOST", "").strip() + if not docker_host or docker_host.startswith("unix://"): + return "127.0.0.1" + + match = re.match(r"^tcp://(.+?):\d+$", docker_host) + if not match: + raise ValueError('Invalid value for DOCKER_HOST: "%s".' % (docker_host,)) + return match.group(1) + + +def load_config(config_path: str) -> Mapping[str, Any]: + with open(f"{os.path.dirname(__file__)}/configs/{config_path}", "r") as config: + return json.load(config) diff --git a/airbyte-integrations/connectors/source-gcs/metadata.yaml b/airbyte-integrations/connectors/source-gcs/metadata.yaml index 99e313b919d3..e069ac0e6a69 100644 --- a/airbyte-integrations/connectors/source-gcs/metadata.yaml +++ b/airbyte-integrations/connectors/source-gcs/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: file connectorType: source definitionId: 2a8c41ae-8c23-4be0-a73f-2ab10ca1a820 - dockerImageTag: 0.7.0 + dockerImageTag: 0.7.1 dockerRepository: airbyte/source-gcs documentationUrl: https://docs.airbyte.com/integrations/sources/gcs githubIssueLabel: source-gcs @@ -30,6 +30,7 @@ data: - cdk:python-file-based connectorTestSuitesOptions: - suite: unitTests + - suite: integrationTests - suite: liveTests testConnections: - name: gcs_old_config_dev_null diff --git a/airbyte-integrations/connectors/source-gcs/poetry.lock b/airbyte-integrations/connectors/source-gcs/poetry.lock index af0bd8b4a581..0e234d13901c 100644 --- a/airbyte-integrations/connectors/source-gcs/poetry.lock +++ b/airbyte-integrations/connectors/source-gcs/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "airbyte-cdk" @@ -450,115 +450,105 @@ files = [ [[package]] name = "cramjam" -version = "2.8.3" +version = "2.8.4" description = "Thin Python bindings to de/compression algorithms in Rust" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "cramjam-2.8.3-cp310-cp310-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:8c8aa6d08c135ae7f0da01e6559a332c5d8fe4989a594db401040e385d04dffd"}, - {file = "cramjam-2.8.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:bd8c601fe8717e52517a2f2eef78217086acf449627bfdda97e3f53fd79c92af"}, - {file = "cramjam-2.8.3-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:dac42b2b4c3950e7eda9b5551e0e904784ed0c0428accc29171c230fb919ec72"}, - {file = "cramjam-2.8.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab8146faa5d8c52edf23724843c36469fc32ff2c4a174eba72f4da6de5016688"}, - {file = "cramjam-2.8.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cb5f4d061e9abdc6663551446c332a58c101efb31fd1746229872600274c2b20"}, - {file = "cramjam-2.8.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5d1ac94e00c64258330105473c641441db02b4dc3e9e9f2963d204e53ed93025"}, - {file = "cramjam-2.8.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ed658f36a2bf667d5b8c7c6690103ad99f81cc62a1b64891b69298447329d4b"}, - {file = "cramjam-2.8.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f6303c8cc583dfe5054cf84717674f75b18bca4ae8e576dc863958d5494dc4b"}, - {file = "cramjam-2.8.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:04b31d427a8902e5c2eec4b8f29873de7a3ade202e3d68e7f2354b9f0aa00bc7"}, - {file = "cramjam-2.8.3-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:9728861bc0390681824961778b36f7f0b95039e8b90d46f1b67f51232f1ee159"}, - {file = "cramjam-2.8.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:87e26e3e1d5fed1cac5b41be648d0daf0793f94cf4a7aebefce1f4f6656e2d21"}, - {file = "cramjam-2.8.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4c1d2d39c2193a77c5e5b327944f90e6ecf2caa1b55e7176cc83d80706ea15de"}, - {file = "cramjam-2.8.3-cp310-none-win32.whl", hash = "sha256:6721edd8f911ad84db83ee4902b7579fc01c55849062f3f1f4171b58fccf98eb"}, - {file = "cramjam-2.8.3-cp310-none-win_amd64.whl", hash = "sha256:4f7c16d358df366e308137411125a2bb50d1b19924fced3a390898fa8c9a074d"}, - {file = "cramjam-2.8.3-cp311-cp311-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:24c2b426dd8fafb894f93a88f42e2827e14199d66836cb100582037e5371c724"}, - {file = "cramjam-2.8.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:007aa9444cb27b8691baae73ca907133cd939987438f874774011b4c740732dd"}, - {file = "cramjam-2.8.3-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:29987b54e31efed66738e8f236c597c4c9a91ec9d57bcb74307712e07505b4bb"}, - {file = "cramjam-2.8.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65bfd41aa92c0025f32ba09214b48e9367a81122586b2617439b4327c4bd179c"}, - {file = "cramjam-2.8.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7337bd8218bd8508f35904274a38cce843a237fe6e23104238bbeb2f337107ed"}, - {file = "cramjam-2.8.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:269f94d2efe6b6a97624782cd3b541e60535dd5874f4a8d5d0ba66ef59424ae3"}, - {file = "cramjam-2.8.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bec9ca5431c32ba94996b7c1c56695b37d48713b97ee1d2a456f4046f009e82f"}, - {file = "cramjam-2.8.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2cb64a97e625ca029b55e37769b8c354e64cbea042c75471915dc385935d30ed"}, - {file = "cramjam-2.8.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c28830ecf76501356d678dac4f37563554ec1c651a53a990cdf595f7ed75c651"}, - {file = "cramjam-2.8.3-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:35647a0e37a4dfec85a44c7966ae476b7db0e6cd65d91c08f1fb3007ed774d92"}, - {file = "cramjam-2.8.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e954599c6369f429a868852eff453b894d88866acba439b65131ea93f5400b47"}, - {file = "cramjam-2.8.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:86e238b6de79e045f5197df2c9dfaf8d10b37a6517ff4ffc4775fe5a3cf4d4a4"}, - {file = "cramjam-2.8.3-cp311-none-win32.whl", hash = "sha256:fe6434d3ee0899bc9396801d1abbc5d1fe77662bd3d1f1c1573fac6708459138"}, - {file = "cramjam-2.8.3-cp311-none-win_amd64.whl", hash = "sha256:e8ec1d4f27eb9d0412f0c567e7ffd14fbeb2b318a1ac394d5de4047c431fe94c"}, - {file = "cramjam-2.8.3-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:24990be4010b2185dcecc67133cd727657036e7b132d7de598148f5b1eb8e452"}, - {file = "cramjam-2.8.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:572cb9a8dc5a189691d6e03a9bf9b4305fd9a9f36bb0f9fde55fc36837c2e6b3"}, - {file = "cramjam-2.8.3-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9efe6915aa7ef176f3a7f42a4e46504573215953331b139abefd20d07d8aba82"}, - {file = "cramjam-2.8.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe84440100e7045190da7f80219be9989b0b6db6acadb3ae9cfe0935d93ebf8c"}, - {file = "cramjam-2.8.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:00524bb23f4abb3a3bfff08aa32b9274843170c5b43855807e0f59670e2ac98c"}, - {file = "cramjam-2.8.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ab67f29094165f0771acad8dd16e840259cfedcc94067af229530496dbf1a24c"}, - {file = "cramjam-2.8.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be6fb5dd5bf1c89c717a73a1057505959f35c08e0e97a76d4cc6391b90d2263b"}, - {file = "cramjam-2.8.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d93b42d22bf3e17290c5e4cf58e715a419330bb5255c35933c14db82ecf3872c"}, - {file = "cramjam-2.8.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:afa065bab70e27565695441f69f493af3d379b8723030f2c3d2547d2e312a4be"}, - {file = "cramjam-2.8.3-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:832224f52fa1e601e0ab678dba9bdfde3686fc4cd1a9f2ed4748f29eaf1cb553"}, - {file = "cramjam-2.8.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:962b7106287bcc463150766b5b8c69f32dcc69713a8dbce00e0ca6936f95c55b"}, - {file = "cramjam-2.8.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2be92c6f0bcffaf8ea6a8164fe0388a188fec2fa9eff1828e8b64dc3a83740f9"}, - {file = "cramjam-2.8.3-cp312-none-win32.whl", hash = "sha256:080f3eb7b648f5ba9d35084d8dddc68246a8f365df239792f6712908f0aa568e"}, - {file = "cramjam-2.8.3-cp312-none-win_amd64.whl", hash = "sha256:c14728e3360cd212d5b606ca703c3bd1c8912efcdbc1aa032c81c2882509ebd5"}, - {file = "cramjam-2.8.3-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:c7e8329cde48740df8d332dade2f52b74612b8ea86005341c99bb192c82a5ce7"}, - {file = "cramjam-2.8.3-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:77346ac669f5445d14b74476a4e8f3a259fd22681bd73790e92b8956d7e225fc"}, - {file = "cramjam-2.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:274878883e7fadf95a6b5bc58f9c1dd39fef2c31d68e18a0fb8594226457fba7"}, - {file = "cramjam-2.8.3-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7871e1fd3ee8ca16799ba22d49fc1e52e78976fa8c659be41630eeb2914475a7"}, - {file = "cramjam-2.8.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:345a952c5d4b922830efaa67dc0b42d21e18c182c1a1bda6d20bb78235f31d6f"}, - {file = "cramjam-2.8.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fb5d7739e2bc573ade12327ef7717b1ac5876c62938fab20eb54d762da23cae2"}, - {file = "cramjam-2.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:440a18fd4ae42e06dbbd7aee91d8248b61da9fef7610ffbd553d1ba93931394b"}, - {file = "cramjam-2.8.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:476890974229713fc7b4c16fb050b756ba926c67e4d1200b3e03c5c051e9b552"}, - {file = "cramjam-2.8.3-cp37-cp37m-musllinux_1_1_armv7l.whl", hash = "sha256:771b44e549f90b5532508782e25d1c40b8054dd83d52253d05945fc05836b252"}, - {file = "cramjam-2.8.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:d824fd98364bc946c38ed324a3ec7befba055285aaf2c1ca61894bb7616226e8"}, - {file = "cramjam-2.8.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2476828dea4089aa3cb9160391f8b36f793ca651afdcba80de1e341373928397"}, - {file = "cramjam-2.8.3-cp37-none-win32.whl", hash = "sha256:4a554bcfd068e831affd64a4f067c7c9b00b359742597c4fdadd18ff673baf30"}, - {file = "cramjam-2.8.3-cp37-none-win_amd64.whl", hash = "sha256:246f1f7d32cac2b64617d2dddba11a82851e73cdcf9d1abb799b08dcd9d2ea49"}, - {file = "cramjam-2.8.3-cp38-cp38-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:bc8f24c32124bb47536882c6b941cdb88cc16e4fa64d5bf347cb8dd72a193fc3"}, - {file = "cramjam-2.8.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:28c30078effc100739d3f9b227276a8360c1b32aac65efb4f641630552213548"}, - {file = "cramjam-2.8.3-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ef0173fb457f73cf9c2553092419db0eba4d582890db95e542a4d93e11340421"}, - {file = "cramjam-2.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a1943f2cc0deee037ddcf92beff6049e12d4e6d557f568ddf59fb3b848f2152"}, - {file = "cramjam-2.8.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5023a737d8d9cf5d123e6d87d088929c3cfb2aae90e0f584204427f74882150a"}, - {file = "cramjam-2.8.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6eec7e985f35708c234542721863d82781d0f7f6a71b45e14ce6d2625d4b131d"}, - {file = "cramjam-2.8.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b188e750b95172c01defcfcfbba629cad797718b34402ec61b3bc9ff99403599"}, - {file = "cramjam-2.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30e2d745cd4d244b7973d15aaebeedb537b980f9d3da80e6dea75ee1a872f9fa"}, - {file = "cramjam-2.8.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c9d54a4aa475d5e902f2ee518bdaa02f26c089e9f72950d00d1643c090f0deb3"}, - {file = "cramjam-2.8.3-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:19b8c97350c8d65daea26267dd1becb59073569aac2ae5743952d7f48da5d37a"}, - {file = "cramjam-2.8.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3277fd42399755d6d3730edec4a192174ee64d219e0ffbc90613f15cbabf711f"}, - {file = "cramjam-2.8.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1fd25201f1278dc6faa2ae35e67b7a5bb352b7fc6ed1ee939637414ca8115863"}, - {file = "cramjam-2.8.3-cp38-none-win32.whl", hash = "sha256:594477faff7f4380fa123cfbcf10ab8ee5af1a28b95750b66931ffafcb11ab5c"}, - {file = "cramjam-2.8.3-cp38-none-win_amd64.whl", hash = "sha256:8ea1dc11538842ff20d9872a17214994f5913cbf3be5594b54aad2422becdf19"}, - {file = "cramjam-2.8.3-cp39-cp39-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:6379b92912f7569e126bd48d10e7087ddd20ea88a939532e3c4a85c2fa05d600"}, - {file = "cramjam-2.8.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:11d2e9eebc7d202eda0ae09fb56a2cdbeb5a1563e89d2118bf18cf0030f35f77"}, - {file = "cramjam-2.8.3-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d5a0a2fe240c97587df07f3d5e1027673d599b3a6a7a0ab540aea69f09e9ff7a"}, - {file = "cramjam-2.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba542f07fe3f41475d78626973533539e6cf2d5b6af37923fe6c7e7f0f74b9b2"}, - {file = "cramjam-2.8.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1374fe9a4431e546bb4501a16b84875d0bf80fc4e6c8942f0d5608ae48474267"}, - {file = "cramjam-2.8.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dcf7791e1cedb982ccc873ec9392c6cfb9c714a64ebf1ed4e8310b9cb44655f2"}, - {file = "cramjam-2.8.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:990e65c2bf1c155a9ddec5ecabf431cf77596432f697d3c6e0831b5174c51c40"}, - {file = "cramjam-2.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9b244d04cef82872d12c227a2f202f080a454d664c05db351626e6ad4aaa307"}, - {file = "cramjam-2.8.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:80b088d15866b37851fd53e2b471becc9ec487257dceca1878621072a18e833e"}, - {file = "cramjam-2.8.3-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:f667843e7a8fca208eecfe44e04088242f8ca60d74d4950fac3722043538d700"}, - {file = "cramjam-2.8.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6f838d06d06709b9ce8b1ceae36aea4e1c7e613365185a91edcbeb5884f5e606"}, - {file = "cramjam-2.8.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4822eb5fe6839cd3d0439e5431e766ad010b2a388ca9617aa6372b6030897782"}, - {file = "cramjam-2.8.3-cp39-none-win32.whl", hash = "sha256:67e09b42e744efd08b93ac56f6100a859a31617d7146725516f3f2c744149d97"}, - {file = "cramjam-2.8.3-cp39-none-win_amd64.whl", hash = "sha256:11c9d30bc53892c57a3b296756c23659323ab1419a2b4bf22bbafc07b247bb67"}, - {file = "cramjam-2.8.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:51e847dcfe74fba379fed2bc2b45f5c2f11c3ece5e9eebcf63f39a9594184588"}, - {file = "cramjam-2.8.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07af94191f6a245226dc8a8bc6c94808e382ce9dfcca4bab0e8015fbc7fc3322"}, - {file = "cramjam-2.8.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc9c45469914099897c47bfc501616fb377f28a865adebf90ea6f3c8ae6dd4e6"}, - {file = "cramjam-2.8.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:ef29fb916fe74be65d0ab8871ab8d964b0f5eb8028bb84b325be43675a59d6e7"}, - {file = "cramjam-2.8.3-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:3850dac9a2f6dcb3249d23f9d505117643b967bdc1c572ed0cc492a48fd69daf"}, - {file = "cramjam-2.8.3-pp310-pypy310_pp73-musllinux_1_1_i686.whl", hash = "sha256:e23e323ad28ed3e4e3a24ceffdab0ff235954109a88b536ea7b3b7886bd0a536"}, - {file = "cramjam-2.8.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1ba1a8ff855b30b4069a9b45ea9e7f2b5d882c7953bdfccda8d4b275fa7057ce"}, - {file = "cramjam-2.8.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:eea606b01b43b91626e3aafd463bd19b6ed739bdb8b2b309e5d7ff72afc0e89d"}, - {file = "cramjam-2.8.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:97c706c520c3f8b0184278cc86187528458350216c6e4fa85d3f16bcad0d365d"}, - {file = "cramjam-2.8.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d08f1bab949ffd6dd6f25a89e4f7062d147aeea9c067e4dd155bdb190e5a519"}, - {file = "cramjam-2.8.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba1e45074757ab0482ac544e60613b6b8658100ac9985c91868a4598cdfb63ba"}, - {file = "cramjam-2.8.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:a2fededed05a042f093dbf1b11d69afb1874a2c9197fcf1d58c142ba9111db5a"}, - {file = "cramjam-2.8.3-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:fc0c6eb8185c68f79a25bb298825e345cc09b826f5828bd8146e3600ca6e9981"}, - {file = "cramjam-2.8.3-pp39-pypy39_pp73-musllinux_1_1_i686.whl", hash = "sha256:6653c262ad71e6c0ae08eeca3af2ee89ad47483b6312f2c6094518cb77872406"}, - {file = "cramjam-2.8.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:6c04f363cb4b316719421724521432b6e7f6490e5baaaf7692af961c28d0279b"}, - {file = "cramjam-2.8.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:e30f1f00de913b440baa36647817b9b7120a69b04eca05f3354aaf5b40f95ee5"}, - {file = "cramjam-2.8.3.tar.gz", hash = "sha256:6b1fa0a6ea8183831d04572597c182bd6cece62d583a36cde1e6a86e72ce2389"}, + {file = "cramjam-2.8.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:e9e112514363261a896f85948d5d055dccaab2a1fa77d440f55030464118a95a"}, + {file = "cramjam-2.8.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ee2441028e813ecc1d10b90640dd2b9649cdefdfe80af1d838cf00fd935ee5e7"}, + {file = "cramjam-2.8.4-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:86a3e0f91176eacd23f8d63b01139a63687cb3fa9670996b3bfa7c38eac6cb7e"}, + {file = "cramjam-2.8.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e34aa083a10079c8814091c0fe9080238a82569fa08058cf79d12b3f9710fc5"}, + {file = "cramjam-2.8.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:465ccf470536e065822daa2a083dedf18df8133278e9132b147bd1721211d707"}, + {file = "cramjam-2.8.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c30aba9e9c737c986d26a809b9e36628452c075234a5e835b085ab7c2b9574dc"}, + {file = "cramjam-2.8.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:52f710bd7fa9b5a374e2e2281d7d672f9eb89263c531643f95fab93e98200c68"}, + {file = "cramjam-2.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6622095ffa6cae77c9e8036a39757fdb1d3cabc3444ad892e5a705882ed06c8d"}, + {file = "cramjam-2.8.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bc618c018594c20696a42faf8a144e1508b8a4312e0d8697f6c64b337e37e5d9"}, + {file = "cramjam-2.8.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:30c75259f58583f96ad9cef7202c70cd6604a9dabf9834211df48a27ec85f84a"}, + {file = "cramjam-2.8.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b9b4bbe7ef3318b2f2aed2a8a658b401a9ad9314d50372f9bb97cdef093f326"}, + {file = "cramjam-2.8.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d7a7c10fb2602d7c8c4dbe4eeacf352477cc1af939fd3537f4e1cd42526855b8"}, + {file = "cramjam-2.8.4-cp310-none-win32.whl", hash = "sha256:831ee2424b095f51c9719b0479d9b413bc849e47160b904a7a8e4a8dcf41d2f7"}, + {file = "cramjam-2.8.4-cp310-none-win_amd64.whl", hash = "sha256:4f6bf5752a0322cc63f955343c390253034b609d167930584bb392bf4179c444"}, + {file = "cramjam-2.8.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d48fd69224a2f4df187856021f545a65486575cba92bb32a14ccad1ce54584a9"}, + {file = "cramjam-2.8.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c53d8dce609607370f01a5db65c79db75db08e9e89cbb9c2a2212b7a3c0b8af3"}, + {file = "cramjam-2.8.4-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d08b8ff282545ab3a414db845e430320555ff7a7eb90517b2c9554e24ca0d763"}, + {file = "cramjam-2.8.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac5fb30cf6c03f72397ead8584592dc071f486c76199c46c28e7de619174ba1f"}, + {file = "cramjam-2.8.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0d86cfb2b457a337db4b7c8cf6a9dafc018806750f28b3c27d71b94e2d4379d0"}, + {file = "cramjam-2.8.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59565a16ce0c71993d3947bdf9301e0d69866c15f37d67d2875809eca998d841"}, + {file = "cramjam-2.8.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6741544b372ba3e6c65db1c44b1a75e48743d091b76a09d7d832b1fb0a0ef518"}, + {file = "cramjam-2.8.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5f486bacd46f364137f5b164a879821115118d7f866a838429eb10aee59a14b"}, + {file = "cramjam-2.8.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4e02081bfb9998f5ff816f3e984a62ca91835e3483c578812374aaf5cb6ed921"}, + {file = "cramjam-2.8.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:1c63e856727726a1ee2a77a12bfccfcd70ee3e5bbe9e6d07bd00be5a1eb6ec10"}, + {file = "cramjam-2.8.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:74fb59137946b691e8987349e9117e2897f3b0484116ad6e2b1b4de0d082430f"}, + {file = "cramjam-2.8.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c7952e0cd6f37a04983cb027175f91f225d7c30625038b8930b6fd3f00619350"}, + {file = "cramjam-2.8.4-cp311-none-win32.whl", hash = "sha256:2bfd5c442e6031b146a93b1cc37d42c04b6d01bb652c9f123338c482c3943038"}, + {file = "cramjam-2.8.4-cp311-none-win_amd64.whl", hash = "sha256:73c95cae138bc8f5604bbbc97860f158c4f77e046304dd4f9c9838021d64217a"}, + {file = "cramjam-2.8.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:5056f476917d31c69719883bbe12272288b77ab5ea5ee55fbcbb6c0dd10e52da"}, + {file = "cramjam-2.8.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8359d31dca4bd8286e031f1a21f20f62f4e7a4586c407e916fd2de101c719a8b"}, + {file = "cramjam-2.8.4-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a1aee32556b9f7ecc61c6c4675798153ac511b5b72db9f56d2a8c20c1fa6d563"}, + {file = "cramjam-2.8.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:005bfe79ae38ea1df67fd3079089287640c780bf112aab4b6a3a9f12f0bf3c91"}, + {file = "cramjam-2.8.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:51662c79c5a2256824f3acca9ccdbeaad3626c90ae46a19ef25f186d70a9ac69"}, + {file = "cramjam-2.8.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c744148e33002cefd4aaa3641800c0008fa177c8c09230c09d30d6e7ab473a4"}, + {file = "cramjam-2.8.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c897d2443cf9f3685a51ecc28c669aad95b6a610de7883647fe450cc742e2ea7"}, + {file = "cramjam-2.8.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:741b0c29d701d470243b9cad09a3e21c2ab83190710df680fd84baea1b262089"}, + {file = "cramjam-2.8.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4cfc6d838afb90a59d2c721fe8d78c2a333edf5c370b3ce8f9823c49bc52e5d0"}, + {file = "cramjam-2.8.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:977e380a567f1bcdb0f1156820fedc57727c6c639769b846b39ad7fc1be5563b"}, + {file = "cramjam-2.8.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3f16dea7f430bb8a5cf2e2a8eece5fa7a6e58bffae3913083f6c20de50ce85bd"}, + {file = "cramjam-2.8.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9d25c2ff722e66a55c58b6c325985b2bf342a6592db084557c2956a07d7179d7"}, + {file = "cramjam-2.8.4-cp312-none-win32.whl", hash = "sha256:b63bcf4e5f9c6ee027947a22862d054e8ce0fa189a33ccdb07e66ef09291252c"}, + {file = "cramjam-2.8.4-cp312-none-win_amd64.whl", hash = "sha256:72b9d4c29a51a8656690df2ef6f7823fa27ebc35e051182b6ebef5fef180876f"}, + {file = "cramjam-2.8.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:9b00949104594eb2b6daf9ec72f1a6dfc93968bc0ffbdbfee936c359fc782186"}, + {file = "cramjam-2.8.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:24b29d805e860d22499e6f5d004582477f3c8309e2a899e0c86c1530a94e6092"}, + {file = "cramjam-2.8.4-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f9454207624a701cb518fbef137e2eb6088aaf5606679aa6ab28d2dd06d72702"}, + {file = "cramjam-2.8.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ee580acb4b6af5ae211b80b679aa377ffa9f9ff74a1e9de458c09d19bce4433"}, + {file = "cramjam-2.8.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:157c36731e430670be44ba490b8a0e4fc04ebdd78c3ea19339ba4ac24d73ad25"}, + {file = "cramjam-2.8.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30a12b1437e28b5e72ab10642d214e9b42220e8c5be2948ac6916aca203f69b0"}, + {file = "cramjam-2.8.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:553e2cd4c2614510770ff3a8bf6b72957a86985b1ae2b8fcdc6d04848857313f"}, + {file = "cramjam-2.8.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e40e448d50fa7c2b79c06d99459ad4a77d58d9cfb3f0549a63b91179a5e57c0b"}, + {file = "cramjam-2.8.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:769995dfc7fd042ce123f25e7659977ed4aa4d5d6aad976970b12b9b4019c116"}, + {file = "cramjam-2.8.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:1ba26f563d9e5be588c8e5f5523b4cdb5b63e3ac3fb28857af9611eb5ea51416"}, + {file = "cramjam-2.8.4-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:5cbfd6c44c85216b3535095258b506f6e246c6fbf1438a79f71bcff4d98f7e3f"}, + {file = "cramjam-2.8.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:4bf4e8711b27604b3ca7e7c224a70f5abe94f5bf05a183bd97677e9cffd2be04"}, + {file = "cramjam-2.8.4-cp313-none-win32.whl", hash = "sha256:7c9ca8e6c33c06c08e9945a20fe0f64a2bcd363554e359a2936b3a469883630a"}, + {file = "cramjam-2.8.4-cp313-none-win_amd64.whl", hash = "sha256:ee92df7e66b7cbdb05b18687a42696bc729bacaad0d68f5549e30cbfa1eb0ca8"}, + {file = "cramjam-2.8.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:386eb0fe9567ae3c06e2053205e19e671e4170f3a0deb68dd103e4c651a3ff8b"}, + {file = "cramjam-2.8.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:64e22027874ce429ce04c0c9d19e6bed5bf6425ecc3e68752211b8509915c57c"}, + {file = "cramjam-2.8.4-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:b74470fb215a3ac2e6ed968f671286456030882aa25616b969b1a52ebda4f29d"}, + {file = "cramjam-2.8.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c41d4542cc2c7238017caebc161b0866b3fb5e85e59727ab623f95e07abc453"}, + {file = "cramjam-2.8.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:14b6f2f883068873bd2b5c31fbf7c4223c0452b8bff662bec02d7973a095c46b"}, + {file = "cramjam-2.8.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5921c4521d41fb125d31ce1fe9e5bfba24a2577bc8727289baae9afbebc8409"}, + {file = "cramjam-2.8.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eb62855f17be5d1bec0d3cef89d8d54582137529c7ea96480c40ebb4a8c92c4b"}, + {file = "cramjam-2.8.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91cd4b28fc75680616bd22db5a56802ce7ce406052c58e72fd583a16746a1010"}, + {file = "cramjam-2.8.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f03502eaf1a0a95cdcbf4c6ebba5edfaa68d356f487ec8485ae651772c9426f9"}, + {file = "cramjam-2.8.4-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:bb5e23c1f8dc2b4cddc7982da60d2f7a9719920539c26e7b754f2272f510fc0c"}, + {file = "cramjam-2.8.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ef6b0d4c83b173d18398713522bff1db1e4e73ec3b3da6495afc5628767d6c85"}, + {file = "cramjam-2.8.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b2253287a08759cefb75ef46ebaa0f993a2890a31fe9bba94363ca245f42d821"}, + {file = "cramjam-2.8.4-cp38-none-win32.whl", hash = "sha256:8375090e54978ccbb1d90e494d73d09e36477e0d695ddadf2d13627168862950"}, + {file = "cramjam-2.8.4-cp38-none-win_amd64.whl", hash = "sha256:12100dd3ed6969365d1952832e39c017d97c85eeb517ae468092f67aa4d89568"}, + {file = "cramjam-2.8.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:47c1594346dceb0d363d479ddac1e9ff87596c92e5258b389118ae6e30599145"}, + {file = "cramjam-2.8.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5472f9c6db34046c7ab2f0c4be5a4be677dba98bf78cc0eb03f9812e5774f14d"}, + {file = "cramjam-2.8.4-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9bfa940e016bfeea2b93115abf9e4e455a6325dd85a3fa6af55c6052f070ba25"}, + {file = "cramjam-2.8.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24e738a92ac520b26b952bfc48b1ba6453ea455e20167f08f6ee3df5c7d22cd4"}, + {file = "cramjam-2.8.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:78ded70e85722a6dcd0c436193af58a43083f0ece35c1f74227782a28e517aa0"}, + {file = "cramjam-2.8.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99b024a9912a5fd3b4e6b949b83b291e2828775edc0595ef8b94c491e904287b"}, + {file = "cramjam-2.8.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:673dc6395fed94db59fb75a7657d8b061bd575332d8f15025e7b1a4feaba0a3f"}, + {file = "cramjam-2.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4b8f83c5a98fecf44c6d852a9bd30ab1508e51d910dc9c8e636863d131fd5eb"}, + {file = "cramjam-2.8.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:85eada9385a706d8d0f6cb1d51995f5eef16d3cade7e68150d6e441fd26406da"}, + {file = "cramjam-2.8.4-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:2429134bb2ee8fffe28f41e3f5390be9c539ac1e2c453034ea63542d7aacc5cc"}, + {file = "cramjam-2.8.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e90003b2ce00358ee669afa0710bf52dee6827460b80ce4a7a9f906551ab703a"}, + {file = "cramjam-2.8.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d0619af45310cceeab9a2410d4a14445743e494015d85584b974847bfb2a2011"}, + {file = "cramjam-2.8.4-cp39-none-win32.whl", hash = "sha256:a30d68094462076655259feee1187237af846969007e5341a96c79b447c47ab3"}, + {file = "cramjam-2.8.4-cp39-none-win_amd64.whl", hash = "sha256:f24e375dfb31f0953e236f2cc4af1b03b80d40aec2bc558df48d507d8e7c8d96"}, + {file = "cramjam-2.8.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3536362f777d817c4994d6eaa42e00e705092c5660fd3d9984f3b0cc6164d327"}, + {file = "cramjam-2.8.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:0d52eabd20a694636f5b0197daa64db497ea518e057935a7c61ec71e92d3ccd6"}, + {file = "cramjam-2.8.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cf69f19ebd546fc155ec3098603de51f52bf620a23597810cb5b34d6aff116d"}, + {file = "cramjam-2.8.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:364258057d579c772e23e1f666fd7efec4f63ea2e791889bb18263c9e9e6aa91"}, + {file = "cramjam-2.8.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:98a2e2c3132b454ae47b194164bb8860464ed410fbbffc0d1de19452cc7cb402"}, + {file = "cramjam-2.8.4.tar.gz", hash = "sha256:ad8bec85b46283330214f4367805e6f56e04ce25a030a2c6a4b127437d006fcf"}, ] [package.extras] -dev = ["black (==22.3.0)", "hypothesis", "numpy", "pytest (>=5.30)", "pytest-xdist"] +dev = ["black (==22.3.0)", "hypothesis", "numpy", "pytest (>=5.30)", "pytest-benchmark", "pytest-xdist"] [[package]] name = "cryptography" @@ -646,6 +636,28 @@ wrapt = ">=1.10,<2" [package.extras] dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] +[[package]] +name = "docker" +version = "7.1.0" +description = "A Python library for the Docker Engine API." +optional = false +python-versions = ">=3.8" +files = [ + {file = "docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0"}, + {file = "docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c"}, +] + +[package.dependencies] +pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""} +requests = ">=2.26.0" +urllib3 = ">=1.26.0" + +[package.extras] +dev = ["coverage (==7.2.7)", "pytest (==7.4.2)", "pytest-cov (==4.1.0)", "pytest-timeout (==2.1.0)", "ruff (==0.1.8)"] +docs = ["myst-parser (==0.18.0)", "sphinx (==5.1.1)"] +ssh = ["paramiko (>=2.4.3)"] +websockets = ["websocket-client (>=1.3.0)"] + [[package]] name = "dpath" version = "2.2.0" @@ -2411,6 +2423,29 @@ files = [ {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, ] +[[package]] +name = "pywin32" +version = "306" +description = "Python for Window Extensions" +optional = false +python-versions = "*" +files = [ + {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, + {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, + {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, + {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, + {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, + {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, + {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, + {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, + {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, + {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, + {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, + {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, + {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, + {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, +] + [[package]] name = "pyyaml" version = "6.0.2" @@ -3218,4 +3253,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10,<3.12" -content-hash = "f8de90fe16b826334107644fb31f7f42af5b08b272c394ca4c5e71dc69c22b92" +content-hash = "d7ed3c601beab220c69932e512a7966c5653b2a7cfec2dd7eb19faa2861a8153" diff --git a/airbyte-integrations/connectors/source-gcs/pyproject.toml b/airbyte-integrations/connectors/source-gcs/pyproject.toml index d60c7d9fe3bb..7850d302c19a 100644 --- a/airbyte-integrations/connectors/source-gcs/pyproject.toml +++ b/airbyte-integrations/connectors/source-gcs/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "0.7.0" +version = "0.7.1" name = "source-gcs" description = "Source implementation for Gcs." authors = [ "Airbyte ",] @@ -26,6 +26,7 @@ airbyte-cdk = {extras = ["file-based"], version = "^5"} source-gcs = "source_gcs.run:run" [tool.poetry.group.dev.dependencies] +docker = "^7.0.0" pytest-mock = "^3.12.0" requests-mock = "^1.11.0" pytest = "^7.4" diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/conftest.py b/airbyte-integrations/connectors/source-gcs/unit_tests/conftest.py index 8ff3dc51e85f..105a55e1ee08 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/conftest.py @@ -24,6 +24,7 @@ def _file_uri() -> str: def remote_file(): return GCSRemoteFile(uri=_file_uri(), last_modified=datetime.now(), mime_type="csv") + @pytest.fixture def remote_file_older(): return GCSRemoteFile(uri=_file_uri(), last_modified=datetime.now() - timedelta(days=1)) diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py index 8fa2b548154b..6b91419574a7 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py @@ -8,5 +8,6 @@ def test_documentation_url(): assert "https" in Config.documentation_url() + def test_remove_discriminator(): assert Config.remove_discriminator({}) is None diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_cursor.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_cursor.py index 96027235fd00..59095eead97d 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_cursor.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_cursor.py @@ -75,8 +75,12 @@ def test_get_files_to_sync_with_existing_older_file(cursor, remote_file, remote_ def test_get_files_to_sync_with_existing_initial_state_and_newer_file(cursor, remote_file, remote_file_older, remote_file_b, logger): - cursor.set_initial_state({"history": {remote_file.uri: remote_file_older.last_modified.replace().strftime(cursor.DATE_TIME_FORMAT)}, - cursor.CURSOR_FIELD: cursor._get_cursor()}) + cursor.set_initial_state( + { + "history": {remote_file.uri: remote_file_older.last_modified.replace().strftime(cursor.DATE_TIME_FORMAT)}, + cursor.CURSOR_FIELD: cursor._get_cursor(), + } + ) DefaultFileBasedCursor.DEFAULT_MAX_HISTORY_SIZE = 1 files_to_sync = list(cursor.get_files_to_sync([remote_file], logger)) @@ -91,7 +95,12 @@ def test_get_files_to_sync_with_no_history(cursor, remote_file, logger): def test_get_files_to_sync_with_same_time_as_initial_state(cursor, remote_file, logger): - cursor.set_initial_state({"history": {remote_file.uri: remote_file.last_modified.strftime(cursor.DATE_TIME_FORMAT)}, cursor.CURSOR_FIELD: cursor._get_cursor()}) + cursor.set_initial_state( + { + "history": {remote_file.uri: remote_file.last_modified.strftime(cursor.DATE_TIME_FORMAT)}, + cursor.CURSOR_FIELD: cursor._get_cursor(), + } + ) files_to_sync = list(cursor.get_files_to_sync([remote_file], logger)) @@ -99,7 +108,12 @@ def test_get_files_to_sync_with_same_time_as_initial_state(cursor, remote_file, def test_get_files_to_sync_with_different_file_as_initial_state(cursor, remote_file, remote_file_older, logger): - cursor.set_initial_state({"history": {remote_file.uri: remote_file.last_modified.strftime(cursor.DATE_TIME_FORMAT)}, cursor.CURSOR_FIELD: cursor._get_cursor()}) + cursor.set_initial_state( + { + "history": {remote_file.uri: remote_file.last_modified.strftime(cursor.DATE_TIME_FORMAT)}, + cursor.CURSOR_FIELD: cursor._get_cursor(), + } + ) DefaultFileBasedCursor.DEFAULT_MAX_HISTORY_SIZE = 1 files_to_sync = list(cursor.get_files_to_sync([remote_file_older], logger)) diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_helpers.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_helpers.py index 66a0c5dc7ff8..7047f82d9317 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_helpers.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_helpers.py @@ -10,4 +10,3 @@ def test_get_stream_name(): blob.name = "path/to/stream.csv" assert get_stream_name(blob) == "stream" - diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_run.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_run.py index 236fafd6c43b..87b46cc061aa 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_run.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_run.py @@ -9,11 +9,14 @@ def test_run_with_non_existing_config(): - with patch("sys.argv", ['', 'check', '--config', 'non_existing.json']): + with patch("sys.argv", ["", "check", "--config", "non_existing.json"]): assert run() is None def test_run_with_invalid_config(): - with patch("sys.argv", ['', 'read', '--config', f'{config_path()}/config_bad_encoding.json', '--catalog', f'{catalog_path()}/catalog_bad_encoding.json']): + with patch( + "sys.argv", + ["", "read", "--config", f"{config_path()}/config_bad_encoding.json", "--catalog", f"{catalog_path()}/catalog_bad_encoding.json"], + ): with pytest.raises(AirbyteTracedException): run() diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_source.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_source.py index f30c503eb118..03062a560865 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_source.py @@ -22,7 +22,9 @@ def _source_gcs(catalog, config): def _mock_encoding_error(): - DefaultFileBasedAvailabilityStrategy.check_availability_and_parsability = Mock(side_effect=AirbyteTracedException(message="encoding error")) + DefaultFileBasedAvailabilityStrategy.check_availability_and_parsability = Mock( + side_effect=AirbyteTracedException(message="encoding error") + ) def test_check_connection_with_airbyte_traced_exception(logger): @@ -48,6 +50,7 @@ def test_check_connection_with_airbyte_traced_exception_multiple_failures(logger assert re.search(r"2 streams with errors.+encoding error", ate.value.message) + def test_read_config_with_invalid_legacy_config(logger): with pytest.raises(ValueError): SourceGCS.read_config(f"{config_path()}/config_legacy.json") diff --git a/docs/integrations/sources/gcs.md b/docs/integrations/sources/gcs.md index b78f00445e58..0d5fb3103a04 100644 --- a/docs/integrations/sources/gcs.md +++ b/docs/integrations/sources/gcs.md @@ -151,44 +151,45 @@ Leaving this field blank (default option) will disallow escaping. | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------| +| 0.7.1 | 2024-09-24 | [45850](https://github.com/airbytehq/airbyte/pull/45850) | Add integration tests | | 0.7.0 | 2024-09-24 | [45671](https://github.com/airbytehq/airbyte/pull/45671) | Add .zip files support | -| 0.6.9 | 2024-09-21 | [45798](https://github.com/airbytehq/airbyte/pull/45798) | Update dependencies | -| 0.6.8 | 2024-09-19 | [45092](https://github.com/airbytehq/airbyte/pull/45092) | Update CDK v5; Fix OSError not raised in stream_reader.open_file | -| 0.6.7 | 2024-09-14 | [45492](https://github.com/airbytehq/airbyte/pull/45492) | Update dependencies | -| 0.6.6 | 2024-09-07 | [45232](https://github.com/airbytehq/airbyte/pull/45232) | Update dependencies | -| 0.6.5 | 2024-08-31 | [45010](https://github.com/airbytehq/airbyte/pull/45010) | Update dependencies | -| 0.6.4 | 2024-08-27 | [44796](https://github.com/airbytehq/airbyte/pull/44796) | Fix empty list of globs when prefix empty | -| 0.6.3 | 2024-08-26 | [44781](https://github.com/airbytehq/airbyte/pull/44781) | Set file signature URL expiration limit default to max | -| 0.6.2 | 2024-08-24 | [44733](https://github.com/airbytehq/airbyte/pull/44733) | Update dependencies | -| 0.6.1 | 2024-08-17 | [44285](https://github.com/airbytehq/airbyte/pull/44285) | Update dependencies | -| 0.6.0 | 2024-08-15 | [44015](https://github.com/airbytehq/airbyte/pull/44015) | Add support for all FileBasedSpec file types | -| 0.5.0 | 2024-08-14 | [44070](https://github.com/airbytehq/airbyte/pull/44070) | Update CDK v4 and Python 3.10 dependencies | -| 0.4.15 | 2024-08-12 | [43733](https://github.com/airbytehq/airbyte/pull/43733) | Update dependencies | -| 0.4.14 | 2024-08-10 | [43512](https://github.com/airbytehq/airbyte/pull/43512) | Update dependencies | -| 0.4.13 | 2024-08-03 | [43236](https://github.com/airbytehq/airbyte/pull/43236) | Update dependencies | -| 0.4.12 | 2024-07-27 | [42693](https://github.com/airbytehq/airbyte/pull/42693) | Update dependencies | -| 0.4.11 | 2024-07-20 | [42312](https://github.com/airbytehq/airbyte/pull/42312) | Update dependencies | -| 0.4.10 | 2024-07-13 | [41865](https://github.com/airbytehq/airbyte/pull/41865) | Update dependencies | -| 0.4.9 | 2024-07-10 | [41430](https://github.com/airbytehq/airbyte/pull/41430) | Update dependencies | -| 0.4.8 | 2024-07-09 | [41148](https://github.com/airbytehq/airbyte/pull/41148) | Update dependencies | -| 0.4.7 | 2024-07-06 | [41015](https://github.com/airbytehq/airbyte/pull/41015) | Update dependencies | -| 0.4.6 | 2024-06-26 | [40540](https://github.com/airbytehq/airbyte/pull/40540) | Update dependencies | -| 0.4.5 | 2024-06-25 | [40391](https://github.com/airbytehq/airbyte/pull/40391) | Update dependencies | -| 0.4.4 | 2024-06-24 | [40234](https://github.com/airbytehq/airbyte/pull/40234) | Update dependencies | -| 0.4.3 | 2024-06-22 | [40089](https://github.com/airbytehq/airbyte/pull/40089) | Update dependencies | -| 0.4.2 | 2024-06-06 | [39255](https://github.com/airbytehq/airbyte/pull/39255) | [autopull] Upgrade base image to v1.2.2 | -| 0.4.1 | 2024-05-29 | [38696](https://github.com/airbytehq/airbyte/pull/38696) | Avoid error on empty stream when running discover | -| 0.4.0 | 2024-03-21 | [36373](https://github.com/airbytehq/airbyte/pull/36373) | Add Gzip and Bzip compression support. Manage dependencies with Poetry. | -| 0.3.7 | 2024-02-06 | [34936](https://github.com/airbytehq/airbyte/pull/34936) | Bump CDK version to avoid missing SyncMode errors | -| 0.3.6 | 2024-01-30 | [34681](https://github.com/airbytehq/airbyte/pull/34681) | Unpin CDK version to make compatible with the Concurrent CDK | -| 0.3.5 | 2024-01-30 | [34661](https://github.com/airbytehq/airbyte/pull/34661) | Pin CDK version until upgrade for compatibility with the Concurrent CDK | -| 0.3.4 | 2024-01-11 | [34158](https://github.com/airbytehq/airbyte/pull/34158) | Fix issue in stream reader for document file type parser | -| 0.3.3 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | -| 0.3.2 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | -| 0.3.1 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | -| 0.3.0 | 2023-10-11 | [31212](https://github.com/airbytehq/airbyte/pull/31212) | Migrated to file based CDK | -| 0.2.0 | 2023-06-26 | [27725](https://github.com/airbytehq/airbyte/pull/27725) | License Update: Elv2 | -| 0.1.0 | 2023-02-16 | [23186](https://github.com/airbytehq/airbyte/pull/23186) | New Source: GCS | +| 0.6.9 | 2024-09-21 | [45798](https://github.com/airbytehq/airbyte/pull/45798) | Update dependencies | +| 0.6.8 | 2024-09-19 | [45092](https://github.com/airbytehq/airbyte/pull/45092) | Update CDK v5; Fix OSError not raised in stream_reader.open_file | +| 0.6.7 | 2024-09-14 | [45492](https://github.com/airbytehq/airbyte/pull/45492) | Update dependencies | +| 0.6.6 | 2024-09-07 | [45232](https://github.com/airbytehq/airbyte/pull/45232) | Update dependencies | +| 0.6.5 | 2024-08-31 | [45010](https://github.com/airbytehq/airbyte/pull/45010) | Update dependencies | +| 0.6.4 | 2024-08-27 | [44796](https://github.com/airbytehq/airbyte/pull/44796) | Fix empty list of globs when prefix empty | +| 0.6.3 | 2024-08-26 | [44781](https://github.com/airbytehq/airbyte/pull/44781) | Set file signature URL expiration limit default to max | +| 0.6.2 | 2024-08-24 | [44733](https://github.com/airbytehq/airbyte/pull/44733) | Update dependencies | +| 0.6.1 | 2024-08-17 | [44285](https://github.com/airbytehq/airbyte/pull/44285) | Update dependencies | +| 0.6.0 | 2024-08-15 | [44015](https://github.com/airbytehq/airbyte/pull/44015) | Add support for all FileBasedSpec file types | +| 0.5.0 | 2024-08-14 | [44070](https://github.com/airbytehq/airbyte/pull/44070) | Update CDK v4 and Python 3.10 dependencies | +| 0.4.15 | 2024-08-12 | [43733](https://github.com/airbytehq/airbyte/pull/43733) | Update dependencies | +| 0.4.14 | 2024-08-10 | [43512](https://github.com/airbytehq/airbyte/pull/43512) | Update dependencies | +| 0.4.13 | 2024-08-03 | [43236](https://github.com/airbytehq/airbyte/pull/43236) | Update dependencies | +| 0.4.12 | 2024-07-27 | [42693](https://github.com/airbytehq/airbyte/pull/42693) | Update dependencies | +| 0.4.11 | 2024-07-20 | [42312](https://github.com/airbytehq/airbyte/pull/42312) | Update dependencies | +| 0.4.10 | 2024-07-13 | [41865](https://github.com/airbytehq/airbyte/pull/41865) | Update dependencies | +| 0.4.9 | 2024-07-10 | [41430](https://github.com/airbytehq/airbyte/pull/41430) | Update dependencies | +| 0.4.8 | 2024-07-09 | [41148](https://github.com/airbytehq/airbyte/pull/41148) | Update dependencies | +| 0.4.7 | 2024-07-06 | [41015](https://github.com/airbytehq/airbyte/pull/41015) | Update dependencies | +| 0.4.6 | 2024-06-26 | [40540](https://github.com/airbytehq/airbyte/pull/40540) | Update dependencies | +| 0.4.5 | 2024-06-25 | [40391](https://github.com/airbytehq/airbyte/pull/40391) | Update dependencies | +| 0.4.4 | 2024-06-24 | [40234](https://github.com/airbytehq/airbyte/pull/40234) | Update dependencies | +| 0.4.3 | 2024-06-22 | [40089](https://github.com/airbytehq/airbyte/pull/40089) | Update dependencies | +| 0.4.2 | 2024-06-06 | [39255](https://github.com/airbytehq/airbyte/pull/39255) | [autopull] Upgrade base image to v1.2.2 | +| 0.4.1 | 2024-05-29 | [38696](https://github.com/airbytehq/airbyte/pull/38696) | Avoid error on empty stream when running discover | +| 0.4.0 | 2024-03-21 | [36373](https://github.com/airbytehq/airbyte/pull/36373) | Add Gzip and Bzip compression support. Manage dependencies with Poetry. | +| 0.3.7 | 2024-02-06 | [34936](https://github.com/airbytehq/airbyte/pull/34936) | Bump CDK version to avoid missing SyncMode errors | +| 0.3.6 | 2024-01-30 | [34681](https://github.com/airbytehq/airbyte/pull/34681) | Unpin CDK version to make compatible with the Concurrent CDK | +| 0.3.5 | 2024-01-30 | [34661](https://github.com/airbytehq/airbyte/pull/34661) | Pin CDK version until upgrade for compatibility with the Concurrent CDK | +| 0.3.4 | 2024-01-11 | [34158](https://github.com/airbytehq/airbyte/pull/34158) | Fix issue in stream reader for document file type parser | +| 0.3.3 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | +| 0.3.2 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | +| 0.3.1 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | +| 0.3.0 | 2023-10-11 | [31212](https://github.com/airbytehq/airbyte/pull/31212) | Migrated to file based CDK | +| 0.2.0 | 2023-06-26 | [27725](https://github.com/airbytehq/airbyte/pull/27725) | License Update: Elv2 | +| 0.1.0 | 2023-02-16 | [23186](https://github.com/airbytehq/airbyte/pull/23186) | New Source: GCS |