Skip to content

Commit

Permalink
test(source-gcs): add integration test (#45850)
Browse files Browse the repository at this point in the history 
Signed-off-by: Artem Inzhyyants <[email protected]>
  • Loading branch information
@artem1205
artem1205 authored Sep 24, 2024
1 parent 2023b64 commit f3c748b
Show file tree
Hide file tree
Showing 30 changed files with 926 additions and 152 deletions.
24 changes: 24 additions & 0 deletions airbyte-integrations/connectors/source-gcs/build_customization.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.


from __future__ import annotations

from typing import TYPE_CHECKING

if TYPE_CHECKING:
from dagger import Container


async def pre_connector_install(base_image_container: Container) -> Container:
"""
Docker compose is required to run the integration tests so we install Docker on top of the base image.
"""
return (
base_image_container.with_exec(["sh", "-c", "apt-get update && apt-get install -y curl jq"])
# Download install-docker.sh script
.with_exec(["curl", "-fsSL", "https://get.docker.com", "-o", "/tmp/install-docker.sh"])
# Run the install-docker.sh script with a pinned Docker version
.with_exec(["sh", "/tmp/install-docker.sh", "--version", "26.0"])
# Remove the install-docker.sh script
.with_exec(["rm", "/tmp/install-docker.sh"])
)
18 changes: 18 additions & 0 deletions ...integrations/connectors/source-gcs/integration_tests/configs/config_integration_avro.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"bucket": "sample-bucket-avro",
"service_account": "{}",
"streams": [
{
"name": "test_avro",
"globs": ["**/*.avro"],
"validation_policy": "Emit Record",
"format": {
"filetype": "avro"
}
}
],
"credentials": {
"service_account": "{}",
"auth_type": "Service"
}
}
18 changes: 18 additions & 0 deletions ...-integrations/connectors/source-gcs/integration_tests/configs/config_integration_csv.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"bucket": "sample-bucket-csv",
"service_account": "{}",
"streams": [
{
"name": "test_csv",
"globs": ["**/*.csv"],
"validation_policy": "Emit Record",
"format": {
"filetype": "csv"
}
}
],
"credentials": {
"service_account": "{}",
"auth_type": "Service"
}
}
18 changes: 18 additions & 0 deletions ...ntegrations/connectors/source-gcs/integration_tests/configs/config_integration_jsonl.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"bucket": "sample-bucket-jsonl",
"service_account": "{}",
"streams": [
{
"name": "test_jsonl",
"globs": ["**/*.jsonl"],
"validation_policy": "Emit Record",
"format": {
"filetype": "jsonl"
}
}
],
"credentials": {
"service_account": "{}",
"auth_type": "Service"
}
}
18 changes: 18 additions & 0 deletions ...egrations/connectors/source-gcs/integration_tests/configs/config_integration_parquet.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"bucket": "sample-bucket-parquet",
"service_account": "{}",
"streams": [
{
"name": "test_parquet",
"globs": ["**/*.parquet"],
"validation_policy": "Emit Record",
"format": {
"filetype": "parquet"
}
}
],
"credentials": {
"service_account": "{}",
"auth_type": "Service"
}
}
35 changes: 35 additions & 0 deletions .../connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_avro.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{
"streams": [
{
"stream": {
"name": "test_avro",
"json_schema": {
"type": "object",
"properties": {
"_ab_source_file_last_modified": {
"type": "string"
},
"_ab_source_file_url": {
"type": "string"
},
"station": {
"type": ["null", "string"]
},
"time": {
"type": ["null", "integer"]
},
"temp": {
"type": ["null", "integer"]
}
}
},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["_ab_source_file_last_modified"],
"is_resumable": true
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
}
]
}
260 changes: 260 additions & 0 deletions ...s/connectors/source-gcs/integration_tests/configured_catalogs/configured_catalog_csv.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,260 @@
{
"streams": [
{
"stream": {
"name": "test_csv",
"json_schema": {
"type": "object",
"properties": {
"_ab_source_file_last_modified": {
"type": "string"
},
"_ab_source_file_url": {
"type": "string"
},
"TOK": {
"type": ["null", "string"]
},
"UPDATE": {
"type": ["null", "string"]
},
"DATE": {
"type": ["null", "string"]
},
"SHOT": {
"type": ["null", "string"]
},
"TIME": {
"type": ["null", "string"]
},
"AUXHEAT": {
"type": ["null", "string"]
},
"PHASE": {
"type": ["null", "string"]
},
"STATE": {
"type": ["null", "string"]
},
"PGASA": {
"type": ["null", "string"]
},
"PGASZ": {
"type": ["null", "string"]
},
"BGASA": {
"type": ["null", "string"]
},
"BGASZ": {
"type": ["null", "string"]
},
"BGASA2": {
"type": ["null", "string"]
},
"BGASZ2": {
"type": ["null", "string"]
},
"PIMPA": {
"type": ["null", "string"]
},
"PIMPZ": {
"type": ["null", "string"]
},
"PELLET": {
"type": ["null", "string"]
},
"RGEO": {
"type": ["null", "string"]
},
"RMAG": {
"type": ["null", "string"]
},
"AMIN": {
"type": ["null", "string"]
},
"SEPLIM": {
"type": ["null", "string"]
},
"XPLIM": {
"type": ["null", "string"]
},
"KAPPA": {
"type": ["null", "string"]
},
"DELTA": {
"type": ["null", "string"]
},
"INDENT": {
"type": ["null", "string"]
},
"AREA": {
"type": ["null", "string"]
},
"VOL": {
"type": ["null", "string"]
},
"CONFIG": {
"type": ["null", "string"]
},
"IGRADB": {
"type": ["null", "string"]
},
"WALMAT": {
"type": ["null", "string"]
},
"DIVMAT": {
"type": ["null", "string"]
},
"LIMMAT": {
"type": ["null", "string"]
},
"EVAP": {
"type": ["null", "string"]
},
"BT": {
"type": ["null", "string"]
},
"IP": {
"type": ["null", "string"]
},
"VSURF": {
"type": ["null", "string"]
},
"Q95": {
"type": ["null", "string"]
},
"BEPMHD": {
"type": ["null", "string"]
},
"BETMHD": {
"type": ["null", "string"]
},
"BEPDIA": {
"type": ["null", "string"]
},
"NEL": {
"type": ["null", "string"]
},
"DNELDT": {
"type": ["null", "string"]
},
"ZEFF": {
"type": ["null", "string"]
},
"PRAD": {
"type": ["null", "string"]
},
"POHM": {
"type": ["null", "string"]
},
"ENBI": {
"type": ["null", "string"]
},
"PINJ": {
"type": ["null", "string"]
},
"BSOURCE": {
"type": ["null", "string"]
},
"PINJ2": {
"type": ["null", "string"]
},
"BSOURCE2": {
"type": ["null", "string"]
},
"COCTR": {
"type": ["null", "string"]
},
"PNBI": {
"type": ["null", "string"]
},
"ECHFREQ": {
"type": ["null", "string"]
},
"ECHMODE": {
"type": ["null", "string"]
},
"ECHLOC": {
"type": ["null", "string"]
},
"PECH": {
"type": ["null", "string"]
},
"ICFREQ": {
"type": ["null", "string"]
},
"ICSCHEME": {
"type": ["null", "string"]
},
"ICANTEN": {
"type": ["null", "string"]
},
"PICRH": {
"type": ["null", "string"]
},
"LHFREQ": {
"type": ["null", "string"]
},
"LHNPAR": {
"type": ["null", "string"]
},
"PLH": {
"type": ["null", "string"]
},
"IBWFREQ": {
"type": ["null", "string"]
},
"PIBW": {
"type": ["null", "string"]
},
"TE0": {
"type": ["null", "string"]
},
"TI0": {
"type": ["null", "string"]
},
"WFANI": {
"type": ["null", "string"]
},
"WFICRH": {
"type": ["null", "string"]
},
"MEFF": {
"type": ["null", "string"]
},
"ISEQ": {
"type": ["null", "string"]
},
"WTH": {
"type": ["null", "string"]
},
"WTOT": {
"type": ["null", "string"]
},
"DWTOT": {
"type": ["null", "string"]
},
"PL": {
"type": ["null", "string"]
},
"PLTH": {
"type": ["null", "string"]
},
"TAUTOT": {
"type": ["null", "string"]
},
"TAUTH": {
"type": ["null", "string"]
}
}
},
"supported_sync_modes": ["full_refresh", "incremental"]
},
"source_defined_cursor": true,
"default_cursor_field": ["_ab_source_file_last_modified"],
"is_resumable": true,
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
}
]
}
Loading

0 comments on commit f3c748b

Please sign in to comment.