From e1d7add50a5da77ac01a253cd1e4603ef8ccb99f Mon Sep 17 00:00:00 2001 From: Jared Rhizor Date: Mon, 9 Nov 2020 11:00:11 -0800 Subject: [PATCH] mostly incremental builds (#817) * support cached builds for base -> base-java -> snowflake * use plugin for image building * fix matching on COPY from * remove docker.gradle * tmp commit * update connectors * finish rest of build files * fix ide errors * more build fixes * clean up * clean up for new soruces * fix spotless * fix flake problems * add recommended empty file * python caching * fixes upon review * clean up docker and build test files * clean up python * clean up * fix integration test dependencies * fix standard tests * fix * remove symlink * re-add requirements to fix normalizatioon build * fix symlink * fix dumbest build problem of all * add missing integration test def * fix missing dep * remove class exclusion * move trim so null source versions are allowed * rename map * fix hardcoded value * remove unnecessary dep * use dashes for salesforce package name * fix typo * DRY and fix test image name * Fix edit * assert string is not empty * build integration test image only for integrationTest * move code generator to tools and rename docker build tasks * make source test depend on integration test build, not the other way * remove guard because the docker build should exist before the integrationtest is applied * remove comment * DRY up airbyte-source-test * fix plugin compilation * add missing dependency * rename getTaggedImage to getDevTaggedImage * fix test vs main docker build bug --- .gitignore | 2 +- .../bases/airbyte-protocol/build.gradle | 15 +- .../code-generator/build.gradle | 2 - .../bases/base-java/build.gradle | 7 +- .../bases/base-normalization/build.gradle | 19 +- .../bases/base-normalization/setup.py | 4 +- .../base_python_test/base_python_test | 1 - .../bases/base-python-test/build.gradle | 16 +- .../bases/base-python/build.gradle | 18 +- .../bases/base-singer/build.gradle | 17 +- airbyte-integrations/bases/base/build.gradle | 4 +- .../bases/destination-test-lib/build.gradle | 1 - .../bases/standard-source-test/build.gradle | 11 +- .../java-destination/build.gradle.hbs | 15 +- .../source-python/build.gradle.hbs | 36 +--- .../source-singer/build.gradle.hbs | 37 +--- .../destination-bigquery/build.gradle | 16 +- .../connectors/destination-csv/build.gradle | 15 +- .../destination-postgres/build.gradle | 15 +- .../destination-snowflake/build.gradle | 15 +- .../build.gradle | 16 +- .../build.gradle | 38 +--- .../connectors/source-file/build.gradle | 31 +-- .../source-github-singer/build.gradle | 28 +-- .../source-google-adwords-singer/build.gradle | 38 +--- .../source-google-sheets/build.gradle | 31 +-- .../airbyte-protocol | 1 + .../airbyte_protocol | 1 - .../build.gradle | 27 +-- .../source-hubspot-singer/build.gradle | 27 +-- .../connectors/source-jdbc/build.gradle | 16 +- .../source-marketo-singer/build.gradle | 37 +--- .../connectors/source-mysql/build.gradle | 16 +- .../connectors/source-postgres/build.gradle | 16 +- .../connectors/source-rest-api/build.gradle | 38 +--- .../source-salesforce-singer/build.gradle | 31 ++- .../source-salesforce-singer/main_dev.py | 2 +- .../source-salesforce-singer/setup.py | 26 +-- .../source_salesforce_singer/__init__.py | 3 +- .../source_salesforce_singer/source.py | 22 +- .../source-stripe-singer/build.gradle | 18 +- build.gradle | 2 +- buildSrc/build.gradle | 11 + buildSrc/settings.gradle | 0 buildSrc/src/main/groovy/DockerHelpers.groovy | 23 +++ .../src/main/groovy/airbyte-docker.gradle | 190 ++++++++++++++++++ .../airbyte-integration-test-java.gradle | 39 ++++ .../src/main/groovy/airbyte-python.gradle | 98 +++++++++ .../main/groovy/airbyte-source-test.gradle | 36 ++++ gradle.properties | 1 + settings.gradle | 16 +- tools/bin/build_image.sh | 17 ++ tools/bin/integration_test_pr.sh | 2 +- tools/bin/standard_test_pr.sh | 2 +- tools/build/README.md | 6 - .../code-generator/Dockerfile | 0 tools/code-generator/build.gradle | 3 + tools/gradle/commons/docker.gradle | 20 -- .../gradle/commons/integrations/image.gradle | 25 --- .../integrations/integration-test.gradle | 29 --- .../gradle/commons/integrations/python.gradle | 48 ----- .../standard-source-test-python.gradle | 41 ---- .../integrations/standard-source-test.gradle | 38 ---- .../commons/integrations/test-image.gradle | 28 --- 64 files changed, 703 insertions(+), 701 deletions(-) delete mode 100644 airbyte-integrations/bases/airbyte-protocol/code-generator/build.gradle delete mode 120000 airbyte-integrations/bases/base-python-test/base_python_test/base_python_test create mode 120000 airbyte-integrations/connectors/source-googleanalytics-singer/airbyte-protocol delete mode 120000 airbyte-integrations/connectors/source-googleanalytics-singer/airbyte_protocol create mode 100644 buildSrc/build.gradle create mode 100644 buildSrc/settings.gradle create mode 100644 buildSrc/src/main/groovy/DockerHelpers.groovy create mode 100644 buildSrc/src/main/groovy/airbyte-docker.gradle create mode 100644 buildSrc/src/main/groovy/airbyte-integration-test-java.gradle create mode 100644 buildSrc/src/main/groovy/airbyte-python.gradle create mode 100644 buildSrc/src/main/groovy/airbyte-source-test.gradle create mode 100755 tools/bin/build_image.sh delete mode 100644 tools/build/README.md rename {airbyte-integrations/bases/airbyte-protocol => tools}/code-generator/Dockerfile (100%) create mode 100644 tools/code-generator/build.gradle delete mode 100644 tools/gradle/commons/docker.gradle delete mode 100644 tools/gradle/commons/integrations/image.gradle delete mode 100644 tools/gradle/commons/integrations/integration-test.gradle delete mode 100644 tools/gradle/commons/integrations/python.gradle delete mode 100644 tools/gradle/commons/integrations/standard-source-test-python.gradle delete mode 100644 tools/gradle/commons/integrations/standard-source-test.gradle delete mode 100644 tools/gradle/commons/integrations/test-image.gradle diff --git a/.gitignore b/.gitignore index 696d67415307..c48ea825361d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,9 @@ .idea *.iml build -!tools/build .DS_Store data +.dockerversions secrets diff --git a/airbyte-integrations/bases/airbyte-protocol/build.gradle b/airbyte-integrations/bases/airbyte-protocol/build.gradle index f0bdfebbc0ae..71c89d74ea73 100644 --- a/airbyte-integrations/bases/airbyte-protocol/build.gradle +++ b/airbyte-integrations/bases/airbyte-protocol/build.gradle @@ -1,14 +1,19 @@ -project.ext.pyModule = 'airbyte_protocol' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') +plugins { + id 'airbyte-python' + id 'airbyte-docker' +} + +airbytePython { + moduleDirectory 'airbyte_protocol' +} task generateProtocolClassFilesWithoutLicense(type: Exec) { environment 'ROOT_DIR', rootDir.absolutePath commandLine 'bin/generate-protocol-files.sh' - dependsOn ':airbyte-integrations:bases:airbyte-protocol:code-generator:buildImage' + dependsOn ':tools:code-generator:airbyteDocker' } +// todo: figure out dependencies task generateProtocolClassFiles(dependsOn: [generateProtocolClassFilesWithoutLicense, rootProject.spotlessPython]) rootProject.spotlessPython.mustRunAfter(generateProtocolClassFilesWithoutLicense) - rootProject.build.dependsOn generateProtocolClassFiles diff --git a/airbyte-integrations/bases/airbyte-protocol/code-generator/build.gradle b/airbyte-integrations/bases/airbyte-protocol/code-generator/build.gradle deleted file mode 100644 index 9e4d2385a40c..000000000000 --- a/airbyte-integrations/bases/airbyte-protocol/code-generator/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') - diff --git a/airbyte-integrations/bases/base-java/build.gradle b/airbyte-integrations/bases/base-java/build.gradle index f3d4780a013e..986c64ff8fcc 100644 --- a/airbyte-integrations/bases/base-java/build.gradle +++ b/airbyte-integrations/bases/base-java/build.gradle @@ -1,14 +1,13 @@ plugins { id 'java-library' + id 'airbyte-docker' } -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') - dependencies { implementation 'commons-cli:commons-cli:1.4' implementation project(':airbyte-config:models') implementation project(':airbyte-protocol:models') -} -buildImage.dependsOn ':airbyte-integrations:bases:base:buildImage' + implementation files(project(':airbyte-integrations:bases:base').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index 74b1f8539ce3..d3b397d445ae 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -1,13 +1,12 @@ -project.ext.pyModule = 'normalization' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') +plugins { + id 'airbyte-docker' + id 'airbyte-python' +} -task unitTest(type: PythonTask){ - command = "setup.py test" +airbytePython { + moduleDirectory 'normalization' } -unitTest.dependsOn(installReqs) -build.dependsOn(unitTest) -build.dependsOn ':airbyte-integrations:bases:airbyte-protocol:build' -buildImage.dependsOn ':airbyte-integrations:bases:base:buildImage' -buildImage.dependsOn ':airbyte-integrations:bases:airbyte-protocol:buildImage' +dependencies { + implementation files(project(':airbyte-integrations:bases:airbyte-protocol').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/bases/base-normalization/setup.py b/airbyte-integrations/bases/base-normalization/setup.py index a8ac4d001789..093c14d2850e 100644 --- a/airbyte-integrations/bases/base-normalization/setup.py +++ b/airbyte-integrations/bases/base-normalization/setup.py @@ -34,11 +34,13 @@ install_requires=["airbyte-protocol", "dbt>=0.18.1", "pyyaml"], package_data={"": ["*.yml"]}, setup_requires=["pytest-runner"], - tests_require=["pytest"], entry_points={ "console_scripts": [ "transform-config=normalization.transform_config.transform:main", "transform-catalog=normalization.transform_catalog.transform:main", ], }, + extras_require={ + "tests": ["airbyte-protocol", "pytest"], + }, ) diff --git a/airbyte-integrations/bases/base-python-test/base_python_test/base_python_test b/airbyte-integrations/bases/base-python-test/base_python_test/base_python_test deleted file mode 120000 index d873dc7a790f..000000000000 --- a/airbyte-integrations/bases/base-python-test/base_python_test/base_python_test +++ /dev/null @@ -1 +0,0 @@ -../../bases/base-python-test/base_python_test \ No newline at end of file diff --git a/airbyte-integrations/bases/base-python-test/build.gradle b/airbyte-integrations/bases/base-python-test/build.gradle index 0e723616e180..f3d0d6e9b812 100644 --- a/airbyte-integrations/bases/base-python-test/build.gradle +++ b/airbyte-integrations/bases/base-python-test/build.gradle @@ -1,6 +1,12 @@ -project.ext.pyModule = 'base_python_test' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') +plugins { + id 'airbyte-docker' + id 'airbyte-python' +} -build.dependsOn(':airbyte-integrations:bases:base-python:build') -buildImage.dependsOn(':airbyte-integrations:bases:base-python:buildImage') +airbytePython { + moduleDirectory 'base_python_test' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/bases/base-python/build.gradle b/airbyte-integrations/bases/base-python/build.gradle index 896e314c0e12..f37c0d27978a 100644 --- a/airbyte-integrations/bases/base-python/build.gradle +++ b/airbyte-integrations/bases/base-python/build.gradle @@ -1,7 +1,13 @@ -project.ext.pyModule = 'base_python' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') +plugins { + id 'airbyte-docker' + id 'airbyte-python' +} -build.dependsOn ':airbyte-integrations:bases:airbyte-protocol:build' -buildImage.dependsOn ':airbyte-integrations:bases:base:buildImage' -buildImage.dependsOn ':airbyte-integrations:bases:airbyte-protocol:buildImage' +airbytePython { + moduleDirectory 'base_python' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:airbyte-protocol').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:base').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/bases/base-singer/build.gradle b/airbyte-integrations/bases/base-singer/build.gradle index b8f6f22d3efc..491458e7d9a1 100644 --- a/airbyte-integrations/bases/base-singer/build.gradle +++ b/airbyte-integrations/bases/base-singer/build.gradle @@ -1,7 +1,12 @@ -project.ext.pyModule = 'base_singer' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') +plugins { + id 'airbyte-docker' + id 'airbyte-python' +} -build.dependsOn ':airbyte-integrations:bases:airbyte-protocol:build' -build.dependsOn ':airbyte-integrations:bases:base-python:build' -buildImage.dependsOn ':airbyte-integrations:bases:base-python:buildImage' +airbytePython { + moduleDirectory 'base_singer' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/bases/base/build.gradle b/airbyte-integrations/bases/base/build.gradle index 09edeab5bc10..c5810d7529af 100644 --- a/airbyte-integrations/bases/base/build.gradle +++ b/airbyte-integrations/bases/base/build.gradle @@ -1 +1,3 @@ -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') +plugins { + id 'airbyte-docker' +} diff --git a/airbyte-integrations/bases/destination-test-lib/build.gradle b/airbyte-integrations/bases/destination-test-lib/build.gradle index 556a46ed0033..ed054f2ce079 100644 --- a/airbyte-integrations/bases/destination-test-lib/build.gradle +++ b/airbyte-integrations/bases/destination-test-lib/build.gradle @@ -8,5 +8,4 @@ dependencies { implementation 'org.junit.jupiter:junit-jupiter-api:5.4.2' implementation 'org.junit.jupiter:junit-jupiter-params:5.4.2' - } diff --git a/airbyte-integrations/bases/standard-source-test/build.gradle b/airbyte-integrations/bases/standard-source-test/build.gradle index 1bc815500ca9..90e8c7032758 100644 --- a/airbyte-integrations/bases/standard-source-test/build.gradle +++ b/airbyte-integrations/bases/standard-source-test/build.gradle @@ -1,8 +1,11 @@ plugins { id 'application' + id 'airbyte-docker' } -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') +application { + mainClass = 'io.airbyte.integrations.base.TestPythonSourceMain' +} dependencies { implementation project(':airbyte-config:models') @@ -15,9 +18,3 @@ dependencies { implementation 'org.junit.platform:junit-platform-launcher:1.7.0' implementation 'org.junit.jupiter:junit-jupiter-api:5.4.2' } - -application { - mainClass = 'io.airbyte.integrations.base.TestPythonSourceMain' -} - -buildImage.dependsOn(assemble) diff --git a/airbyte-integrations/connector-templates/java-destination/build.gradle.hbs b/airbyte-integrations/connector-templates/java-destination/build.gradle.hbs index 77f656fc4053..8a875b5f9de9 100644 --- a/airbyte-integrations/connector-templates/java-destination/build.gradle.hbs +++ b/airbyte-integrations/connector-templates/java-destination/build.gradle.hbs @@ -1,9 +1,12 @@ plugins { id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' } -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') +application { + mainClass = 'io.airbyte.integrations.destination.{{snakeCase name}}.{{properCase name}}Destination' +} dependencies { implementation project(':airbyte-config:models') @@ -11,12 +14,6 @@ dependencies { implementation project(':airbyte-integrations:bases:base-java') integrationTestImplementation project(':airbyte-integrations:bases:destination-test-lib') -} -application { - mainClass = 'io.airbyte.integrations.destination.{{snakeCase name}}.{{properCase name}}Destination' + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } - -buildImage.dependsOn(assemble) -buildImage.dependsOn(':airbyte-integrations:bases:base-java:buildImage') -integrationTest.dependsOn(buildImage) diff --git a/airbyte-integrations/connector-templates/source-python/build.gradle.hbs b/airbyte-integrations/connector-templates/source-python/build.gradle.hbs index bab375244a63..bf4401ea0385 100644 --- a/airbyte-integrations/connector-templates/source-python/build.gradle.hbs +++ b/airbyte-integrations/connector-templates/source-python/build.gradle.hbs @@ -1,33 +1,13 @@ -project.ext.pyModule = 'source_{{snakeCase name}}' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/test-image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test-python.gradle') - - -standardSourceTestPython { - ext { - imageName = "${extractImageName(project.file('Dockerfile'))}:dev" - pythonContainerName = "${extractImageName(project.file('Dockerfile.test'))}:dev" - } +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-test' } -task installTestDeps(type: PythonTask){ - module = "pip" - command = "install .[tests]" +airbytePython { + moduleDirectory 'source_{{snakeCase name}}' } -task unitTest(type: PythonTask){ - module = "pytest" - command = "unit_tests" +dependencies { + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } - -unitTest.dependsOn(installTestDeps) -build.dependsOn(unitTest) -build.dependsOn ':airbyte-integrations:bases:base-python-test:build' -buildImage.dependsOn ':airbyte-integrations:bases:base-python:buildImage' -integrationTest.dependsOn(buildImage) - -buildTestImage.dependsOn ':airbyte-integrations:bases:base-python-test:buildImage' -standardSourceTestPython.dependsOn(buildTestImage) diff --git a/airbyte-integrations/connector-templates/source-singer/build.gradle.hbs b/airbyte-integrations/connector-templates/source-singer/build.gradle.hbs index 6dedfb42ff8f..b75ae9810a20 100644 --- a/airbyte-integrations/connector-templates/source-singer/build.gradle.hbs +++ b/airbyte-integrations/connector-templates/source-singer/build.gradle.hbs @@ -1,34 +1,13 @@ -project.ext.pyModule = 'source_{{snakeCase name}}_singer' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/test-image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test-python.gradle') - - -standardSourceTestPython { - ext { - imageName = "${extractImageName(project.file('Dockerfile'))}:dev" - pythonContainerName = "${extractImageName(project.file('Dockerfile.test'))}:dev" - } +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-test' } -task installTestDeps(type: PythonTask){ - module = "pip" - command = "install .[tests]" +airbytePython { + moduleDirectory 'source_{{snakeCase name}}_singer' } -task unitTest(type: PythonTask){ - module = "pytest" - command = "unit_tests" +dependencies { + implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } - -unitTest.dependsOn(installTestDeps) -build.dependsOn(unitTest) -build.dependsOn ':airbyte-integrations:bases:base-python-test:build' - -buildImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage' -buildTestImage.dependsOn ':airbyte-integrations:bases:base-python-test:buildImage' - -integrationTest.dependsOn(buildImage) -standardSourceTestPython.dependsOn(buildTestImage) diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index 78ab5bbce830..f5db8f5f0f11 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -1,9 +1,12 @@ plugins { id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' } -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') +application { + mainClass = 'io.airbyte.integrations.destination.bigquery.BigQueryDestination' +} dependencies { implementation 'com.google.cloud:google-cloud-bigquery:1.122.2' @@ -16,12 +19,5 @@ dependencies { integrationTestImplementation project(':airbyte-integrations:bases:destination-test-lib') + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } - -application { - mainClass = 'io.airbyte.integrations.destination.bigquery.BigQueryDestination' -} - -buildImage.dependsOn(assemble) -buildImage.dependsOn(':airbyte-integrations:bases:base-java:buildImage') -integrationTest.dependsOn(buildImage) diff --git a/airbyte-integrations/connectors/destination-csv/build.gradle b/airbyte-integrations/connectors/destination-csv/build.gradle index 4d2b74dc5826..f9c2e30e541d 100644 --- a/airbyte-integrations/connectors/destination-csv/build.gradle +++ b/airbyte-integrations/connectors/destination-csv/build.gradle @@ -1,9 +1,12 @@ plugins { id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' } -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') +application { + mainClass = 'io.airbyte.integrations.destination.csv.CsvDestination' +} dependencies { implementation 'org.apache.commons:commons-csv:1.4' @@ -12,12 +15,6 @@ dependencies { implementation project(':airbyte-integrations:bases:base-java') integrationTestImplementation project(':airbyte-integrations:bases:destination-test-lib') -} -application { - mainClass = 'io.airbyte.integrations.destination.csv.CsvDestination' + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } - -buildImage.dependsOn(assemble) -buildImage.dependsOn(':airbyte-integrations:bases:base-java:buildImage') -integrationTest.dependsOn(buildImage) diff --git a/airbyte-integrations/connectors/destination-postgres/build.gradle b/airbyte-integrations/connectors/destination-postgres/build.gradle index da15a5e7a115..9e7304004e4b 100644 --- a/airbyte-integrations/connectors/destination-postgres/build.gradle +++ b/airbyte-integrations/connectors/destination-postgres/build.gradle @@ -1,9 +1,12 @@ plugins { id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' } -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') +application { + mainClass = 'io.airbyte.integrations.destination.postgres.PostgresDestination' +} dependencies { implementation project(':airbyte-db') @@ -15,12 +18,6 @@ dependencies { integrationTestImplementation project(':airbyte-integrations:bases:destination-test-lib') integrationTestImplementation "org.testcontainers:postgresql:1.15.0-rc2" -} -application { - mainClass = 'io.airbyte.integrations.destination.postgres.PostgresDestination' + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } - -buildImage.dependsOn(assemble) -buildImage.dependsOn(':airbyte-integrations:bases:base-java:buildImage') -integrationTest.dependsOn(buildImage) diff --git a/airbyte-integrations/connectors/destination-snowflake/build.gradle b/airbyte-integrations/connectors/destination-snowflake/build.gradle index 349e0115a33f..5a86875e34e6 100644 --- a/airbyte-integrations/connectors/destination-snowflake/build.gradle +++ b/airbyte-integrations/connectors/destination-snowflake/build.gradle @@ -1,9 +1,12 @@ plugins { id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' } -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') +application { + mainClass = 'io.airbyte.integrations.destination.snowflake.SnowflakeDestination' +} dependencies { implementation 'net.snowflake:snowflake-jdbc:3.12.14' @@ -17,12 +20,6 @@ dependencies { integrationTestImplementation project(':airbyte-integrations:bases:destination-test-lib') integrationTestImplementation 'org.apache.commons:commons-lang3:3.11' integrationTestImplementation project(':airbyte-integrations:connectors:destination-snowflake') -} -application { - mainClass = 'io.airbyte.integrations.destination.snowflake.SnowflakeDestination' + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } - -buildImage.dependsOn(assemble) -buildImage.dependsOn(':airbyte-integrations:bases:base-java:buildImage') -integrationTest.dependsOn(buildImage) diff --git a/airbyte-integrations/connectors/source-exchangeratesapi-singer/build.gradle b/airbyte-integrations/connectors/source-exchangeratesapi-singer/build.gradle index 8f49ab66bd94..5bc4bae3cf01 100644 --- a/airbyte-integrations/connectors/source-exchangeratesapi-singer/build.gradle +++ b/airbyte-integrations/connectors/source-exchangeratesapi-singer/build.gradle @@ -1,18 +1,18 @@ plugins { id 'java' + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-integration-test-java' + // todo: implement source test } -project.ext.pyModule = 'source_exchangeratesapi_singer' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') +airbytePython { + moduleDirectory 'source_exchangeratesapi_singer' +} dependencies { integrationTestImplementation project(':airbyte-workers') integrationTestImplementation project(':airbyte-config:models') integrationTestImplementation project(':airbyte-protocol:models') + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } - -integrationTest.dependsOn(buildImage) -build.dependsOn ':airbyte-integrations:bases:base-singer:build' -buildImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage' diff --git a/airbyte-integrations/connectors/source-facebook-marketing-api-singer/build.gradle b/airbyte-integrations/connectors/source-facebook-marketing-api-singer/build.gradle index 493301341475..ef0ab89be65e 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing-api-singer/build.gradle +++ b/airbyte-integrations/connectors/source-facebook-marketing-api-singer/build.gradle @@ -1,35 +1,13 @@ -project.ext.pyModule = 'source_facebook_marketing_api_singer' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/test-image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test-python.gradle') - - -standardSourceTestPython { - ext { - imageName = "${extractImageName(project.file('Dockerfile'))}:dev" - pythonContainerName = "${extractImageName(project.file('Dockerfile.test'))}:dev" - } +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-test' } -task installTestDeps(type: PythonTask){ - module = "pip" - command = "install .[tests]" +airbytePython { + moduleDirectory 'source_facebook_marketing_api_singer' } -task unitTest(type: PythonTask){ - module = "pytest" - command = "unit_tests" +dependencies { + implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } - -unitTest.dependsOn(installTestDeps) -build.dependsOn(unitTest) -build.dependsOn ':airbyte-integrations:bases:base-python-test:build' - -buildImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage' -integrationTest.dependsOn(buildImage) - -buildTestImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage' -buildTestImage.dependsOn ':airbyte-integrations:bases:base-python-test:buildImage' -standardSourceTestPython.dependsOn(buildTestImage) diff --git a/airbyte-integrations/connectors/source-file/build.gradle b/airbyte-integrations/connectors/source-file/build.gradle index 9fad4ec945d0..3c2b5939b77a 100644 --- a/airbyte-integrations/connectors/source-file/build.gradle +++ b/airbyte-integrations/connectors/source-file/build.gradle @@ -1,26 +1,13 @@ -project.ext.pyModule = 'source_file' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/test-image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test-python.gradle') - - -standardSourceTestPython { - ext { - imageName = "${extractImageName(project.file('Dockerfile'))}:dev" - pythonContainerName = "${extractImageName(project.file('Dockerfile.test'))}:dev" - } +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-test' } -task unitTest(type: PythonTask){ - command = "setup.py test" +airbytePython { + moduleDirectory 'source_file' } -build.dependsOn(unitTest) -build.dependsOn ':airbyte-integrations:bases:base-python-test:build' -buildImage.dependsOn ':airbyte-integrations:bases:base-python:buildImage' -integrationTest.dependsOn(buildImage) - -buildTestImage.dependsOn ':airbyte-integrations:bases:base-python-test:buildImage' -standardSourceTestPython.dependsOn(buildTestImage) +dependencies { + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-github-singer/build.gradle b/airbyte-integrations/connectors/source-github-singer/build.gradle index 30c378280b78..2601ceee71aa 100644 --- a/airbyte-integrations/connectors/source-github-singer/build.gradle +++ b/airbyte-integrations/connectors/source-github-singer/build.gradle @@ -1,24 +1,14 @@ plugins { - id 'java' + id 'airbyte-docker' + id 'airbyte-integration-test-java' + id 'airbyte-python' + id 'airbyte-source-test' } -project.ext.pyModule = 'source_github_singer' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/test-image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test-python.gradle') - -standardSourceTestPython { - ext { - imageName = "${extractImageName(project.file('Dockerfile'))}:dev" - pythonContainerName = "${extractImageName(project.file('Dockerfile.test'))}:dev" - } +airbytePython { + moduleDirectory 'source_github_singer' } -standardSourceTestPython.dependsOn(buildTestImage) -build.dependsOn ':airbyte-integrations:bases:base-singer:build' -build.dependsOn ':airbyte-integrations:bases:base-python-test:build' -buildImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage' -buildTestImage.dependsOn ':airbyte-integrations:bases:base-python-test:buildImage' -integrationTest.dependsOn(buildImage) +dependencies { + implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle b/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle index c0a9a655b292..6106950749cf 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle +++ b/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle @@ -1,34 +1,14 @@ -project.ext.pyModule = 'source_google_adwords_singer' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/test-image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test-python.gradle') - - -standardSourceTestPython { - ext { - imageName = "${extractImageName(project.file('Dockerfile'))}:dev" - pythonContainerName = "${extractImageName(project.file('Dockerfile.test'))}:dev" - } +plugins { + id 'java' + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-test' } -task installTestDeps(type: PythonTask){ - module = "pip" - command = "install .[tests]" +airbytePython { + moduleDirectory 'source_google_adwords_singer' } -task unitTest(type: PythonTask){ - module = "pytest" - command = "unit_tests" +dependencies { + implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } - -unitTest.dependsOn(installTestDeps) -build.dependsOn(unitTest) -build.dependsOn ':airbyte-integrations:bases:base-python-test:build' - -buildImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage' -buildTestImage.dependsOn ':airbyte-integrations:bases:base-python-test:buildImage' - -integrationTest.dependsOn(buildImage) -standardSourceTestPython.dependsOn(buildTestImage) diff --git a/airbyte-integrations/connectors/source-google-sheets/build.gradle b/airbyte-integrations/connectors/source-google-sheets/build.gradle index c8a90436082b..264b9aaafa3d 100644 --- a/airbyte-integrations/connectors/source-google-sheets/build.gradle +++ b/airbyte-integrations/connectors/source-google-sheets/build.gradle @@ -1,30 +1,13 @@ plugins { - id 'java' + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-test' } -project.ext.pyModule = 'google_sheets_source' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/test-image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test-python.gradle') - - -standardSourceTestPython { - ext { - imageName = "${extractImageName(project.file('Dockerfile'))}:dev" - pythonContainerName = "${extractImageName(project.file('Dockerfile.test'))}:dev" - } +airbytePython { + moduleDirectory 'google_sheets_source' } -task unitTest(type: PythonTask){ - command = "setup.py test" +dependencies { + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } - -build.dependsOn(unitTest) -build.dependsOn ':airbyte-integrations:bases:base-python-test:build' -buildImage.dependsOn ':airbyte-integrations:bases:base-python:buildImage' -integrationTest.dependsOn(buildImage) - -buildTestImage.dependsOn ':airbyte-integrations:bases:base-python-test:buildImage' -standardSourceTestPython.dependsOn(buildTestImage) diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/airbyte-protocol b/airbyte-integrations/connectors/source-googleanalytics-singer/airbyte-protocol new file mode 120000 index 000000000000..77d93de614c8 --- /dev/null +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/airbyte-protocol @@ -0,0 +1 @@ +../../bases/airbyte-protocol \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/airbyte_protocol b/airbyte-integrations/connectors/source-googleanalytics-singer/airbyte_protocol deleted file mode 120000 index 124c03ba9ebb..000000000000 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/airbyte_protocol +++ /dev/null @@ -1 +0,0 @@ -../../bases/base-python/airbyte_protocol \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle b/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle index e203c5b6a38d..b91f9928da4a 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle @@ -1,20 +1,13 @@ -project.ext.pyModule = 'source_googleanalytics_singer' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/test-image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test-python.gradle') +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-test' +} -standardSourceTestPython { - ext { - imageName = "${extractImageName(project.file('Dockerfile'))}:dev" - pythonContainerName = "${extractImageName(project.file('Dockerfile.test'))}:dev" - } +airbytePython { + moduleDirectory 'source_googleanalytics_signger' } -standardSourceTestPython.dependsOn(buildTestImage) -build.dependsOn ':airbyte-integrations:bases:base-singer:build' -buildImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage' -build.dependsOn ':airbyte-integrations:bases:base-python-test:build' -buildTestImage.dependsOn ':airbyte-integrations:bases:base-python-test:buildImage' -integrationTest.dependsOn(buildImage) +dependencies { + implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-hubspot-singer/build.gradle b/airbyte-integrations/connectors/source-hubspot-singer/build.gradle index d88fa050983b..4867fb90e768 100644 --- a/airbyte-integrations/connectors/source-hubspot-singer/build.gradle +++ b/airbyte-integrations/connectors/source-hubspot-singer/build.gradle @@ -1,20 +1,13 @@ -project.ext.pyModule = 'source_hubspot_singer' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/test-image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test-python.gradle') +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-test' +} -standardSourceTestPython { - ext { - imageName = "${extractImageName(project.file('Dockerfile'))}:dev" - pythonContainerName = "${extractImageName(project.file('Dockerfile.test'))}:dev" - } +airbytePython { + moduleDirectory 'source_hubspot_singer' } -standardSourceTestPython.dependsOn(buildTestImage) -build.dependsOn ':airbyte-integrations:bases:base-singer:build' -buildImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage' -build.dependsOn ':airbyte-integrations:bases:base-python-test:build' -buildTestImage.dependsOn ':airbyte-integrations:bases:base-python-test:buildImage' -integrationTest.dependsOn(buildImage) +dependencies { + implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-jdbc/build.gradle b/airbyte-integrations/connectors/source-jdbc/build.gradle index 137f6502edfd..731f038bc6d6 100644 --- a/airbyte-integrations/connectors/source-jdbc/build.gradle +++ b/airbyte-integrations/connectors/source-jdbc/build.gradle @@ -1,9 +1,13 @@ plugins { id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' + id 'airbyte-source-test' } -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') +application { + mainClass = 'io.airbyte.integrations.source.jdbc.JdbcSource' +} dependencies { implementation project(':airbyte-db') @@ -15,12 +19,6 @@ dependencies { integrationTestImplementation project(':airbyte-integrations:bases:standard-source-test') integrationTestImplementation "org.testcontainers:postgresql:1.15.0-rc2" -} -application { - mainClass = 'io.airbyte.integrations.source.jdbc.JdbcSource' + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } - -buildImage.dependsOn(assemble) -buildImage.dependsOn(':airbyte-integrations:bases:base-java:buildImage') -integrationTest.dependsOn(buildImage) diff --git a/airbyte-integrations/connectors/source-marketo-singer/build.gradle b/airbyte-integrations/connectors/source-marketo-singer/build.gradle index d9df3e793bfe..03f195e8d50e 100644 --- a/airbyte-integrations/connectors/source-marketo-singer/build.gradle +++ b/airbyte-integrations/connectors/source-marketo-singer/build.gradle @@ -1,34 +1,13 @@ -project.ext.pyModule = 'source_marketo_singer' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/test-image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test-python.gradle') - - -standardSourceTestPython { - ext { - imageName = "${extractImageName(project.file('Dockerfile'))}:dev" - pythonContainerName = "${extractImageName(project.file('Dockerfile.test'))}:dev" - } +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-test' } -task installTestDeps(type: PythonTask){ - module = "pip" - command = "install .[tests]" +airbytePython { + moduleDirectory 'source_marketo_singer' } -task unitTest(type: PythonTask){ - module = "pytest" - command = "unit_tests" +dependencies { + implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } - -unitTest.dependsOn(installTestDeps) -build.dependsOn(unitTest) -build.dependsOn ':airbyte-integrations:bases:base-python-test:build' - -buildImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage' -buildTestImage.dependsOn ':airbyte-integrations:bases:base-python-test:buildImage' - -integrationTest.dependsOn(buildImage) -standardSourceTestPython.dependsOn(buildTestImage) diff --git a/airbyte-integrations/connectors/source-mysql/build.gradle b/airbyte-integrations/connectors/source-mysql/build.gradle index 6780fd64ed88..8c6e9f8de0f9 100644 --- a/airbyte-integrations/connectors/source-mysql/build.gradle +++ b/airbyte-integrations/connectors/source-mysql/build.gradle @@ -1,9 +1,13 @@ plugins { id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' + id 'airbyte-source-test' } -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') +application { + mainClass = 'io.airbyte.integrations.source.mysql.MySqlSource' +} dependencies { implementation project(':airbyte-db') @@ -20,12 +24,6 @@ dependencies { integrationTestImplementation project(':airbyte-integrations:bases:standard-source-test') testImplementation 'org.testcontainers:mysql:1.15.0-rc2' -} -application { - mainClass = 'io.airbyte.integrations.source.mysql.MySqlSource' + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } - -buildImage.dependsOn(assemble) -buildImage.dependsOn(':airbyte-integrations:bases:base-java:buildImage') -integrationTest.dependsOn(buildImage) diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index 5dcc0d6254d4..998d4ea8abc3 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -1,9 +1,13 @@ plugins { id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' + // todo: needs standard source test } -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') +application { + mainClass = 'io.airbyte.integrations.source.postgres.PostgresSource' +} dependencies { implementation "org.postgresql:postgresql:42.2.18" @@ -21,12 +25,6 @@ dependencies { testImplementation project(':airbyte-test-utils') integrationTestImplementation project(':airbyte-integrations:bases:standard-source-test') -} -application { - mainClass = 'io.airbyte.integrations.source.postgres.PostgresSource' + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } - -buildImage.dependsOn(assemble) -buildImage.dependsOn(':airbyte-integrations:bases:base-java:buildImage') -integrationTest.dependsOn(buildImage) diff --git a/airbyte-integrations/connectors/source-rest-api/build.gradle b/airbyte-integrations/connectors/source-rest-api/build.gradle index fb531dcd6ecd..55b9ff925d62 100644 --- a/airbyte-integrations/connectors/source-rest-api/build.gradle +++ b/airbyte-integrations/connectors/source-rest-api/build.gradle @@ -1,33 +1,15 @@ -project.ext.pyModule = 'source_rest_api' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/test-image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test-python.gradle') - - -standardSourceTestPython { - ext { - imageName = "${extractImageName(project.file('Dockerfile'))}:dev" - pythonContainerName = "${extractImageName(project.file('Dockerfile.test'))}:dev" - } +plugins { + id 'java' + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-integration-test-java' + id 'airbyte-source-test' } -task installTestDeps(type: PythonTask){ - module = "pip" - command = "install .[tests]" +airbytePython { + moduleDirectory 'source_rest_api' } -task unitTest(type: PythonTask){ - module = "pytest" - command = "unit_tests" +dependencies { + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } - -unitTest.dependsOn(installTestDeps) -build.dependsOn(unitTest) -build.dependsOn ':airbyte-integrations:bases:base-python-test:build' -buildImage.dependsOn ':airbyte-integrations:bases:base-python:buildImage' -integrationTest.dependsOn(buildImage) - -buildTestImage.dependsOn ':airbyte-integrations:bases:base-python-test:buildImage' -standardSourceTestPython.dependsOn(buildTestImage) diff --git a/airbyte-integrations/connectors/source-salesforce-singer/build.gradle b/airbyte-integrations/connectors/source-salesforce-singer/build.gradle index 9a4e2e02deaf..f05522336848 100644 --- a/airbyte-integrations/connectors/source-salesforce-singer/build.gradle +++ b/airbyte-integrations/connectors/source-salesforce-singer/build.gradle @@ -1,25 +1,20 @@ plugins { - id 'java' + id 'airbyte-docker' + id 'airbyte-integration-test-java' + id 'airbyte-python' } -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') -//apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test.gradle') - -dependencies { +airbytePython { + moduleDirectory 'source_salesforce_singer' } -// We cannot run these tests with every build because it blows through the Salesforce quota. For now, turn this on as needed and run these tests locally; they will not be run in CI. +// We cannot run standard source tests with every build because it blows through the Salesforce quota. For now, turn this on as needed and run these tests locally; they will not be run in CI. // CRITICAL Salesforce has reported 13060/15000 (87.07%) total REST quota used across all Salesforce Applications. Terminating replication to not continue past configured percentage of 80% total quota. -//standardSourceTest { -// ext { -// imageName = "${extractImageName(project.file('Dockerfile'))}:dev" -// specPath = 'source_salesforce_singer/spec.json' -// configPath ='secrets/config.json' -// catalogPath = 'standard_test/catalog.json' -// } -//} +// imageName = "${extractImageName(project.file('Dockerfile'))}:dev" +// specPath = 'source_salesforce_singer/spec.json' +// configPath ='secrets/config.json' +// catalogPath = 'standard_test/catalog.json' -build.dependsOn ':airbyte-integrations:bases:base-singer:build' -buildImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage' -integrationTest.dependsOn(buildImage) +dependencies { + implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-salesforce-singer/main_dev.py b/airbyte-integrations/connectors/source-salesforce-singer/main_dev.py index 9e248457f6c9..6f0631721721 100644 --- a/airbyte-integrations/connectors/source-salesforce-singer/main_dev.py +++ b/airbyte-integrations/connectors/source-salesforce-singer/main_dev.py @@ -23,8 +23,8 @@ """ import sys -from airbyte_protocol.entrypoint import launch +from airbyte_protocol.entrypoint import launch from source_salesforce_singer import SourceSalesforceSinger if __name__ == "__main__": diff --git a/airbyte-integrations/connectors/source-salesforce-singer/setup.py b/airbyte-integrations/connectors/source-salesforce-singer/setup.py index 85976857cae1..682f4964b19d 100644 --- a/airbyte-integrations/connectors/source-salesforce-singer/setup.py +++ b/airbyte-integrations/connectors/source-salesforce-singer/setup.py @@ -22,23 +22,19 @@ SOFTWARE. """ -from setuptools import setup, find_packages +from setuptools import find_packages, setup setup( - name='source_salesforce_singer', - description='Source implementation for Salesforce.', - author='Airbyte', - author_email='contact@airbyte.io', - + name="source-salesforce-singer", + description="Source implementation for Salesforce.", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), - package_data={ - '': ['*.json'] - }, - + package_data={"": ["*.json"]}, install_requires=[ - 'tap-salesforce==1.4.34', - 'requests', - 'airbyte-protocol', - 'base-singer', - ] + "tap-salesforce==1.4.34", + "requests", + "airbyte-protocol", + "base-singer", + ], ) diff --git a/airbyte-integrations/connectors/source-salesforce-singer/source_salesforce_singer/__init__.py b/airbyte-integrations/connectors/source-salesforce-singer/source_salesforce_singer/__init__.py index 11ea21a7cdd1..4939d71e2962 100644 --- a/airbyte-integrations/connectors/source-salesforce-singer/source_salesforce_singer/__init__.py +++ b/airbyte-integrations/connectors/source-salesforce-singer/source_salesforce_singer/__init__.py @@ -22,5 +22,6 @@ SOFTWARE. """ -from .source import * +from .source import SourceSalesforceSinger +__all__ = ["SourceSalesforceSinger"] diff --git a/airbyte-integrations/connectors/source-salesforce-singer/source_salesforce_singer/source.py b/airbyte-integrations/connectors/source-salesforce-singer/source_salesforce_singer/source.py index 24a667062c6f..62f029dde07f 100644 --- a/airbyte-integrations/connectors/source-salesforce-singer/source_salesforce_singer/source.py +++ b/airbyte-integrations/connectors/source-salesforce-singer/source_salesforce_singer/source.py @@ -37,16 +37,16 @@ def check(self, logger, config_container) -> AirbyteConnectionStatus: # pulled from tap-salesforce singer impl # https://github.com/singer-io/tap-salesforce/blob/master/tap_salesforce/salesforce/__init__.py#L295-L327 - if json_config['is_sandbox']: - login_url = 'https://test.salesforce.com/services/oauth2/token' + if json_config["is_sandbox"]: + login_url = "https://test.salesforce.com/services/oauth2/token" else: - login_url = 'https://login.salesforce.com/services/oauth2/token' + login_url = "https://login.salesforce.com/services/oauth2/token" login_body = { - 'grant_type': 'refresh_token', - 'client_id': json_config['client_id'], - 'client_secret': json_config['client_secret'], - 'refresh_token': json_config['refresh_token'] + "grant_type": "refresh_token", + "client_id": json_config["client_id"], + "client_secret": json_config["client_secret"], + "refresh_token": json_config["refresh_token"], } logger.info("Attempting login via OAuth2") @@ -54,7 +54,7 @@ def check(self, logger, config_container) -> AirbyteConnectionStatus: r = None try: logger.info(f"Making POST request to {login_url} with body {login_body}") - headers={"Content-Type": "application/x-www-form-urlencoded"} + headers = {"Content-Type": "application/x-www-form-urlencoded"} r = requests.post(login_url, headers=headers, data=login_body) if r.status_code == 200: logger.info("OAuth2 login successful") @@ -64,8 +64,8 @@ def check(self, logger, config_container) -> AirbyteConnectionStatus: except Exception as e: error_message = str(e) - if r is None and hasattr(e, 'response') and e.response is not None: #pylint:disable=no-member - r = e.response #pylint:disable=no-member + if r is None and hasattr(e, "response") and e.response is not None: # pylint:disable=no-member + r = e.response # pylint:disable=no-member # NB: requests.models.Response is always falsy here. It is false if status code >= 400 if isinstance(r, requests.models.Response): error_message = error_message + ", Response from Salesforce: {}".format(r.text) @@ -85,5 +85,5 @@ def read_cmd(self, logger, config_path, catalog_path, state_path=None) -> str: def transform_config(self, raw_config): # the select_fields_by_default is opinionated about schema changes. we want to reserve the right for the Airbyte system to handle these changes, instead of the singer source. rendered_config = dict(raw_config) - rendered_config['select_fields_by_default'] = True + rendered_config["select_fields_by_default"] = True return rendered_config diff --git a/airbyte-integrations/connectors/source-stripe-singer/build.gradle b/airbyte-integrations/connectors/source-stripe-singer/build.gradle index 11188095a5c9..c993ff22038b 100644 --- a/airbyte-integrations/connectors/source-stripe-singer/build.gradle +++ b/airbyte-integrations/connectors/source-stripe-singer/build.gradle @@ -1,11 +1,13 @@ plugins { - id 'java' + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-integration-test-java' + // todo: needs standard source test } -project.ext.pyModule = 'source_stripe_singer' -apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') +airbytePython { + moduleDirectory 'source_stripe_singer' +} dependencies { integrationTestImplementation 'org.apache.commons:commons-dbcp2:2.7.0' @@ -16,8 +18,6 @@ dependencies { integrationTestImplementation project(':airbyte-workers') integrationTestImplementation project(':airbyte-config:models') integrationTestImplementation project(':airbyte-protocol:models') -} -build.dependsOn ':airbyte-integrations:bases:base-singer:build' -buildImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage' -integrationTest.dependsOn(buildImage) + implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) +} diff --git a/build.gradle b/build.gradle index 64736b29038d..b1b2abb7378f 100644 --- a/build.gradle +++ b/build.gradle @@ -2,7 +2,6 @@ plugins { id 'base' id 'pmd' id 'com.diffplug.spotless' version '5.7.0' - id 'ru.vyarus.use-python' version '2.2.0' apply false } repositories { @@ -49,6 +48,7 @@ def createSpotlessTarget = { pattern -> '.mypy_cache', '.venv', '*.egg-info', + 'build', 'dbt-project-template' ] return fileTree(dir: rootDir, include: pattern, exclude: excludes.collect {"**/${it}"}) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle new file mode 100644 index 000000000000..588f58482440 --- /dev/null +++ b/buildSrc/build.gradle @@ -0,0 +1,11 @@ +plugins { + id 'groovy-gradle-plugin' +} + +repositories { + jcenter() +} + +dependencies { + implementation 'ru.vyarus:gradle-use-python-plugin:2.2.0' +} diff --git a/buildSrc/settings.gradle b/buildSrc/settings.gradle new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/buildSrc/src/main/groovy/DockerHelpers.groovy b/buildSrc/src/main/groovy/DockerHelpers.groovy new file mode 100644 index 000000000000..875f6320de9e --- /dev/null +++ b/buildSrc/src/main/groovy/DockerHelpers.groovy @@ -0,0 +1,23 @@ +import java.nio.file.Paths + +class DockerHelpers { + static String extractLabelValue(String dockerFile, String labelName) { + def file = dockerFile instanceof File ? dockerFile : new File(dockerFile) + return file.readLines() + .grep({ it.startsWith('LABEL') && it.contains(labelName) }) + .get(0) + .split('=')[1] + } + + static String extractImageName(String dockerFile) { + return extractLabelValue(dockerFile, "io.airbyte.name") + } + + static String extractImageVersion(String dockerFile) { + return extractLabelValue(dockerFile, "io.airbyte.version") + } + + static String getDevTaggedImage(projectDir, dockerfileName) { + return "${extractImageName(Paths.get(projectDir.absolutePath, dockerfileName).toString())}:dev" + } +} diff --git a/buildSrc/src/main/groovy/airbyte-docker.gradle b/buildSrc/src/main/groovy/airbyte-docker.gradle new file mode 100644 index 000000000000..908c4e0ccf03 --- /dev/null +++ b/buildSrc/src/main/groovy/airbyte-docker.gradle @@ -0,0 +1,190 @@ +import org.gradle.api.DefaultTask +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.file.FileCollection +import org.gradle.api.tasks.CacheableTask +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.TaskAction +import org.slf4j.Logger + +import java.nio.file.Path +import java.security.MessageDigest +import java.nio.file.Paths + +@CacheableTask +abstract class AirbyteDockerTask extends DefaultTask { + @Input + abstract File rootDir + + @InputFiles + abstract FileCollection projectFiles + + @Input + abstract Set baseImageHashes + + @Input + abstract File projectDir + + @Input + String dockerfileName + + @OutputFile + abstract File idFileOutput + + def buildDockerfile(String scriptPath, String fileName) { + if (project.file(fileName).exists()) { + def tag = DockerHelpers.getDevTaggedImage(projectDir, dockerfileName) + + project.exec { + commandLine scriptPath, rootDir.absolutePath, projectDir.absolutePath, dockerfileName, tag, idFileOutput.absolutePath + } + } + } + + @TaskAction + def airbyteDocker() { + def scriptPath = Paths.get(rootDir.absolutePath, 'tools/bin/build_image.sh').toString() + buildDockerfile(scriptPath, dockerfileName) + } +} + +class AirbyteDockerPlugin implements Plugin { + + static def getBaseTaggedImages(File dockerfile) { + def result = [] as Set + + dockerfile.eachLine { line -> + if (line.startsWith("FROM ")) { + def image = line.split()[1] + assert !image.isEmpty() + result.add(image) + } else if (line.startsWith("COPY --from=")) { + def image = line.substring("COPY --from=".length()).split()[0] + assert !image.isEmpty() + result.add(image) + } + } + + return result + } + + static def getBaseImageHashes(Map imageToHash, File dockerfile) { + def result = [] as Set + + getBaseTaggedImages(dockerfile).forEach { taggedImage -> + result.add((String) imageToHash.get(taggedImage)) + } + + return result + } + + static String getImageHash(Project project, String taggedImage) { + def stdout = new ByteArrayOutputStream() + + project.exec { + commandLine "docker", "images", "--no-trunc", "-f", "dangling=false", "--format", "{{.ID}}", taggedImage + standardOutput = stdout; + } + + return "$stdout".toString().trim() + } + + static boolean isUpToDate(Logger logger, File idFileOutput, Project project, String dockerFile, Path dockerPath) { + if (idFileOutput.exists()) { + def taggedImage = DockerHelpers.getDevTaggedImage(project.projectDir, dockerFile) + logger.debug "taggedImage " + taggedImage + + def current = getImageHash(project, taggedImage) + logger.debug "current " + current + def stored = (String) project.rootProject.imageToHash.get(taggedImage) + logger.debug "stored " + stored + + def notUpToDate = new ArrayList(getBaseTaggedImages(dockerPath.toFile())).any { baseImage -> + logger.debug "checking base image " + baseImage + def storedBase = (String) project.rootProject.imageToHash.get(baseImage) + def currentBase = getImageHash(project, baseImage) + + logger.debug "storedBase " + storedBase + logger.debug "currentBase " + currentBase + if (!currentBase.equals(storedBase)) { + logger.debug "did not match" + return true + } else { + logger.debug "did match" + return false + } + } + + if (notUpToDate) { + return false; + } + + logger.debug "stored " + stored + + def upToDate = current.equals(stored) + + logger.debug "uptodate " + upToDate.toString() + + return upToDate + } else { + return false + } + } + + static def createTask(Project project, String taskName, String dockerFile) { + if (project.file(dockerFile).exists()) { + def filteredProjectFiles = project.fileTree(project.projectDir).filter { + file -> !file.toString().contains(".venv") + } + project.task(taskName, type: AirbyteDockerTask) { + def dockerPath = Paths.get(project.projectDir.absolutePath, dockerFile) + def hash = MessageDigest.getInstance("MD5").digest(dockerPath.getBytes()).encodeHex().toString() + dockerfileName = dockerFile + rootDir = project.rootProject.rootDir + projectDir = project.projectDir + projectFiles = filteredProjectFiles + idFileOutput = project.file(Paths.get(project.rootProject.rootDir.absolutePath, '.dockerversions', hash).toString()) + baseImageHashes = getBaseImageHashes(project.rootProject.imageToHash, dockerPath.toFile()) + dependsOn project.assemble + + outputs.upToDateWhen { + return isUpToDate(logger, idFileOutput, project, dockerFile, dockerPath) + } + } + } else { + project.task(taskName) { + logger.info "Skipping ${taskName} because ${dockerFile} does not exist." + } + } + } + + void apply(Project project) { + // set (and cache) global image to version map + project.rootProject.ext.imageToHash = { + if (!project.rootProject.hasProperty("imageToHash")) { + def imageToHash = [:] + def stdout = new ByteArrayOutputStream() + project.exec { + commandLine "docker", "images", "--no-trunc", "-f", "dangling=false", "--format", "{{.Repository}}:{{.Tag}} {{.ID}}" + standardOutput = stdout; + } + + "$stdout".eachLine { line -> + def splits = line.split() + imageToHash.put(splits[0], splits[1].trim()) + } + + return imageToHash + } else { + return project.rootProject.imageToHash + } + }() + + createTask(project, 'airbyteDocker', 'Dockerfile') + createTask(project, 'airbyteDockerTest', 'Dockerfile.test') + + project.build.dependsOn project.airbyteDocker + } +} diff --git a/buildSrc/src/main/groovy/airbyte-integration-test-java.gradle b/buildSrc/src/main/groovy/airbyte-integration-test-java.gradle new file mode 100644 index 000000000000..664d5c6da948 --- /dev/null +++ b/buildSrc/src/main/groovy/airbyte-integration-test-java.gradle @@ -0,0 +1,39 @@ +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.tasks.testing.Test + +class AirbyteIntegrationTestJavaPlugin implements Plugin { + void apply(Project project) { + project.sourceSets { + integrationTest { + java { + srcDir 'src/test-integration/java' + } + resources { + srcDir 'src/test-integration/resources' + } + } + } + project.test.dependsOn('compileIntegrationTestJava') + + project.configurations { + integrationTestImplementation.extendsFrom testImplementation + integrationTestRuntimeOnly.extendsFrom testRuntimeOnly + } + + project.task('integrationTest', type: Test) { + testClassesDirs += project.sourceSets.integrationTest.output.classesDirs + classpath += project.sourceSets.integrationTest.runtimeClasspath + + useJUnitPlatform() + testLogging() { + events "passed", "failed" + exceptionFormat "full" + } + + dependsOn project.airbyteDocker + mustRunAfter project.test + } + + } +} diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle new file mode 100644 index 000000000000..9c31b2769fa6 --- /dev/null +++ b/buildSrc/src/main/groovy/airbyte-python.gradle @@ -0,0 +1,98 @@ +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.tasks.CacheableTask +import ru.vyarus.gradle.plugin.python.task.PythonTask; + +class AirbytePythonConfiguration { + String moduleDirectory +} + +class AirbytePythonPlugin implements Plugin { + + void apply(Project project) { + def extension = project.extensions.create('airbytePython', AirbytePythonConfiguration) + + project.plugins.apply 'ru.vyarus.use-python' + + project.python { + envPath = '.venv' + minPythonVersion = '3.7' + scope = 'VIRTUALENV' + installVirtualenv = true + pip 'flake8:3.8.4' + pip 'black:20.8b1' + pip 'mypy:0.790' + pip 'isort:5.6.4' + pip 'pytest:6.1.2' + } + + project.task('blackFormat', type: PythonTask) { + module = "black" + // the line length should match .isort.cfg + command = ". --line-length 140" + } + + project.task('isortFormat', type: PythonTask) { + module = "isort" + command = ". --settings-file ${project.rootProject.file('tools/python/.isort.cfg').absolutePath}" + } + + project.task('flakeCheck', type: PythonTask, dependsOn: project.blackFormat) { + module = "flake8" + command = ". --config ${project.rootProject.file('tools/python/.flake8').absolutePath}" + } + + project.task('installReqs', type: PythonTask) { + module = "pip" + command = "install -r requirements.txt" + inputs.file('requirements.txt') + outputs.file('build/installedreqs.txt') + outputs.cacheIf { true } + } + + project.task('installTestReqs', type: PythonTask, dependsOn: project.installReqs) { + module = "pip" + command = "install .[tests]" + inputs.file('setup.py') + outputs.file('build/installedtestreqs.txt') + outputs.cacheIf { true } + } + + if(project.file('unit_tests').exists()) { + project.task('unitTest', type: PythonTask) { + module = "pytest" + command = "unit_tests" + } + } else { + project.task('unitTest') { + logger.info "Skipping Python unit tests because unit_tests directory doesn't exist." + } + } + + if (extension.moduleDirectory) { + project.task('mypyCheck', type: PythonTask) { + module = "mypy" + command = "-m ${extension.moduleDirectory} --config-file ${project.rootProject.file('tools/python/.mypy.ini').absolutePath}" + } + + project.check.dependsOn mypyCheck + } + + project.task('airbytePythonApply', type: DefaultTask) { + dependsOn project.installReqs + dependsOn project.blackFormat + dependsOn project.isortFormat + dependsOn project.flakeCheck + } + + + project.task('airbytePythonTest', type: DefaultTask) { + dependsOn project.installTestReqs + dependsOn project.unitTest + } + + project.assemble.dependsOn project.airbytePythonApply + project.assemble.dependsOn project.airbytePythonTest + project.test.dependsOn project.airbytePythonTest + } +} diff --git a/buildSrc/src/main/groovy/airbyte-source-test.gradle b/buildSrc/src/main/groovy/airbyte-source-test.gradle new file mode 100644 index 000000000000..256910e01256 --- /dev/null +++ b/buildSrc/src/main/groovy/airbyte-source-test.gradle @@ -0,0 +1,36 @@ +import org.gradle.api.Plugin +import org.gradle.api.Project + + +class AirbyteSourceTestPlugin implements Plugin { + void apply(Project project) { + project.task('standardSourceTestPython') { + doFirst { + project.exec { + def imageName = DockerHelpers.getDevTaggedImage(project.projectDir, 'Dockerfile') + def pythonContainerName = DockerHelpers.getDevTaggedImage(project.projectDir, 'Dockerfile.test') + logger.info('standard test inputs') + logger.info("imageName: ${imageName}") + logger.info("pythonContainerName: ${pythonContainerName}") + workingDir project.rootDir + commandLine 'docker', 'run', '--rm', '-i', + // so that it has access to docker + '-v', "/var/run/docker.sock:/var/run/docker.sock", + // when launching the container within a container, it mounts the directory from + // the host filesystem, not the parent container. this forces /tmp to be the + // same directory for host, parent container, and child container. + '-v', "/tmp:/tmp", + // mount the project dir. all provided input paths must be relative to that dir. + '-v', "${project.projectDir.absolutePath}:/test_input", + '--name', "std-test-${project.name}", 'airbyte/standard-source-test:dev', + '--imageName', imageName, + '--pythonContainerName', pythonContainerName + } + } + } + project.standardSourceTestPython.dependsOn(':airbyte-integrations:bases:standard-source-test:airbyteDocker') + project.standardSourceTestPython.dependsOn(project.build) + project.standardSourceTestPython.dependsOn(project.airbyteDocker) + project.standardSourceTestPython.dependsOn(project.airbyteDockerTest) + } +} diff --git a/gradle.properties b/gradle.properties index d3898623d853..6b971f664049 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,4 @@ org.gradle.parallel=true org.gradle.vfs.watch=true org.gradle.jvmargs=-Xmx2g +org.gradle.caching=true diff --git a/settings.gradle b/settings.gradle index 9e30a860b537..ccfa355cf6f7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -12,6 +12,7 @@ gradleEnterprise { } import groovy.io.FileType +import java.nio.file.Files rootProject.name = 'airbyte' @@ -32,12 +33,19 @@ include ':airbyte-webapp' include ':airbyte-workers' include ':airbyte-tests' include ':airbyte-test-utils' +include ':tools:code-generator' // include all integration subprojects def integrationsPath = rootDir.toPath().resolve('airbyte-integrations') -integrationsPath.eachFileRecurse(FileType.FILES) { path -> - def relativePath = integrationsPath.relativize(path) - if (path.endsWith('build.gradle') && !path.contains("build") && relativePath.getNameCount() > 1) { - include ":airbyte-integrations:${relativePath.parent.join(':')}" +def symlinkPrefixes = [] as Set +integrationsPath.eachFileRecurse(FileType.ANY) { path -> + if (Files.isSymbolicLink(path)) { + symlinkPrefixes.add(path.toString()) + } else if(!(symlinkPrefixes.any {prefix -> path.toString().startsWith(prefix)})) { + def relativePath = integrationsPath.relativize(path) + + if (path.endsWith('build.gradle') && !path.contains("build") && relativePath.getNameCount() > 1) { + include ":airbyte-integrations:${relativePath.parent.join(':')}" + } } } diff --git a/tools/bin/build_image.sh b/tools/bin/build_image.sh new file mode 100755 index 000000000000..dccd508d03c0 --- /dev/null +++ b/tools/bin/build_image.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -e + +ROOT_DIR="$1" +PROJECT_DIR="$2" +DOCKERFILE="$3" +TAG="$4" +ID_FILE="$5" + +cd "$ROOT_DIR" +. tools/lib/lib.sh +assert_root + +cd "$PROJECT_DIR" + +DOCKER_BUILDKIT=1 docker build -f "$DOCKERFILE" . -t "$TAG" --iidfile "$ID_FILE" diff --git a/tools/bin/integration_test_pr.sh b/tools/bin/integration_test_pr.sh index 0304bc900374..d44e6e8d699b 100755 --- a/tools/bin/integration_test_pr.sh +++ b/tools/bin/integration_test_pr.sh @@ -6,7 +6,7 @@ set -e assert_root -./gradlew --no-daemon --scan -x generateProtocolClassFiles \ +./gradlew --no-daemon --scan \ :airbyte-integrations:connectors:destination-bigquery:integrationTest \ :airbyte-integrations:connectors:destination-postgres:integrationTest \ :airbyte-integrations:connectors:destination-csv:integrationTest \ diff --git a/tools/bin/standard_test_pr.sh b/tools/bin/standard_test_pr.sh index 3b3baaa7d040..a1d80a813f56 100755 --- a/tools/bin/standard_test_pr.sh +++ b/tools/bin/standard_test_pr.sh @@ -6,6 +6,6 @@ set -e assert_root -./gradlew --no-daemon --scan -x generateProtocolClassFiles \ +./gradlew --no-daemon --scan \ :airbyte-integrations:connectors:source-github-singer:standardSourceTestPython \ :airbyte-integrations:connectors:source-marketo-singer:standardSourceTestPython diff --git a/tools/build/README.md b/tools/build/README.md deleted file mode 100644 index 1497abf702fd..000000000000 --- a/tools/build/README.md +++ /dev/null @@ -1,6 +0,0 @@ -This docker image contains the build environment used to generate all our artifacts. - -If you need to do some changes the environment: -1. Bump up the version in the docker file (`io.airbyte.version`). -1. Build and push the image (with the new tag). -1. Update the consumers to use the most recent build image. diff --git a/airbyte-integrations/bases/airbyte-protocol/code-generator/Dockerfile b/tools/code-generator/Dockerfile similarity index 100% rename from airbyte-integrations/bases/airbyte-protocol/code-generator/Dockerfile rename to tools/code-generator/Dockerfile diff --git a/tools/code-generator/build.gradle b/tools/code-generator/build.gradle new file mode 100644 index 000000000000..c5810d7529af --- /dev/null +++ b/tools/code-generator/build.gradle @@ -0,0 +1,3 @@ +plugins { + id 'airbyte-docker' +} diff --git a/tools/gradle/commons/docker.gradle b/tools/gradle/commons/docker.gradle deleted file mode 100644 index 9a219f310f02..000000000000 --- a/tools/gradle/commons/docker.gradle +++ /dev/null @@ -1,20 +0,0 @@ -String extractLabelValue(dockerFile, String labelName) { - def file = dockerFile instanceof File ? dockerFile : new File(dockerFile) - return file.readLines() - .grep({ it.startsWith('LABEL') && it.contains(labelName) }) - .get(0) - .split('=')[1] -} - -String extractImageName(dockerFile) { - return extractLabelValue(dockerFile, "io.airbyte.name") -} - -String extractImageVersion(dockerFile) { - return extractLabelValue(dockerFile, "io.airbyte.version") -} - -ext { - extractImageName = this.&extractImageName - extractImageVersion = this.&extractImageVersion -} diff --git a/tools/gradle/commons/integrations/image.gradle b/tools/gradle/commons/integrations/image.gradle deleted file mode 100644 index a8321c98513e..000000000000 --- a/tools/gradle/commons/integrations/image.gradle +++ /dev/null @@ -1,25 +0,0 @@ -import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage - -buildscript { - repositories { - jcenter() - } - dependencies { - classpath "com.bmuschko:gradle-docker-plugin:6.6.1" - } -} - -apply plugin: com.bmuschko.gradle.docker.DockerRemoteApiPlugin -apply from: rootProject.file('tools/gradle/commons/docker.gradle') - -task buildImage(type: DockerBuildImage) { - inputDir = projectDir - images.add("${extractImageName(project.file('Dockerfile'))}:dev") - - // Handles the case where there's an upstream dependency on a dev versioned image built by Gradle. - // The plugin doesn't recognize changes to the base image as a change that requires rebuilding. - // Docker will still cache the artifact, so it shouldn't be much slower. - outputs.upToDateWhen { false } -} - -buildImage.dependsOn(build) diff --git a/tools/gradle/commons/integrations/integration-test.gradle b/tools/gradle/commons/integrations/integration-test.gradle deleted file mode 100644 index db306a6cdb84..000000000000 --- a/tools/gradle/commons/integrations/integration-test.gradle +++ /dev/null @@ -1,29 +0,0 @@ -sourceSets { - integrationTest { - java { - srcDir 'src/test-integration/java' - } - resources { - srcDir 'src/test-integration/resources' - } - } -} -test.dependsOn('compileIntegrationTestJava') - -configurations { - integrationTestImplementation.extendsFrom testImplementation - integrationTestRuntimeOnly.extendsFrom testRuntimeOnly -} - -task integrationTest(type: Test) { - testClassesDirs += sourceSets.integrationTest.output.classesDirs - classpath += sourceSets.integrationTest.runtimeClasspath - - useJUnitPlatform() - testLogging() { - events "passed", "failed" - exceptionFormat "full" - } - - mustRunAfter test -} diff --git a/tools/gradle/commons/integrations/python.gradle b/tools/gradle/commons/integrations/python.gradle deleted file mode 100644 index 79942d185b61..000000000000 --- a/tools/gradle/commons/integrations/python.gradle +++ /dev/null @@ -1,48 +0,0 @@ -apply plugin: 'ru.vyarus.use-python' - - -python { - envPath = '.venv' - minPythonVersion = '3.7' - scope = VIRTUALENV - installVirtualenv = true - pip 'flake8:3.8.4' - pip 'black:20.8b1' - pip 'mypy:0.790' - pip 'isort:5.6.4' -} - -task installReqs(type: PythonTask) { - module = "pip" - command = "install -r requirements.txt" -} - -task blackFormat(type: PythonTask) { - module = "black" - // the line length should match .isort.cfg - command = ". --line-length 140" -} - -task isortFormat(type: PythonTask) { - module = "isort" - command = ". --settings-file ${rootProject.file('tools/python/.isort.cfg').absolutePath}" -} - -task flakeCheck(type: PythonTask, dependsOn: blackFormat) { - module = "flake8" - command = ". --config ${rootProject.file('tools/python/.flake8').absolutePath}" -} - -if (project.ext.has('pyModule')) { - task mypyCheck(type: PythonTask, dependsOn: installReqs) { - module = "mypy" - command = "-m ${project.ext.pyModule} --config-file ${rootProject.file('tools/python/.mypy.ini').absolutePath}" - } - - check.dependsOn mypyCheck -} - -build.dependsOn installReqs -check.dependsOn blackFormat -check.dependsOn isortFormat -check.dependsOn flakeCheck diff --git a/tools/gradle/commons/integrations/standard-source-test-python.gradle b/tools/gradle/commons/integrations/standard-source-test-python.gradle deleted file mode 100644 index fc0a5022441d..000000000000 --- a/tools/gradle/commons/integrations/standard-source-test-python.gradle +++ /dev/null @@ -1,41 +0,0 @@ -apply from: rootProject.file('tools/gradle/commons/docker.gradle') - -task standardSourceTestPython { - ext { - imageName = '' - pythonContainerName = '' - } - - doFirst { - exec { - println('standard test inputs') - println("imageName: ${imageName}") - println("pythonContainerName: ${pythonContainerName}") - workingDir rootDir - println(""" Running: -docker run --rm -i \\ - -v "/var/run/docker.sock:/var/run/docker.sock" \\ - -v "/tmp:/tmp" \\ - -v "${project.projectDir.absolutePath}:/test_input" \\ - --name "std-test-${project.name}" airbyte/standard-source-test:dev \\ - --imageName ${imageName} \\ - --pythonContainerName ${pythonContainerName} - """) - commandLine 'docker', 'run', '--rm', '-i', - // so that it has access to docker - '-v', "/var/run/docker.sock:/var/run/docker.sock", - // when launching the container within a container, it mounts the directory from - // the host filesystem, not the parent container. this forces /tmp to be the - // same directory for host, parent container, and child container. - '-v', "/tmp:/tmp", - // mount the project dir. all provided input paths must be relative to that dir. - '-v', "${project.projectDir.absolutePath}:/test_input", - '--name', "std-test-${project.name}", 'airbyte/standard-source-test:dev', - '--imageName', imageName, - '--pythonContainerName', pythonContainerName - } - } -} -standardSourceTestPython.dependsOn(':airbyte-integrations:bases:standard-source-test:buildImage') -standardSourceTestPython.dependsOn(compileJava) -standardSourceTestPython.dependsOn(buildImage) diff --git a/tools/gradle/commons/integrations/standard-source-test.gradle b/tools/gradle/commons/integrations/standard-source-test.gradle deleted file mode 100644 index 867c83d7fbff..000000000000 --- a/tools/gradle/commons/integrations/standard-source-test.gradle +++ /dev/null @@ -1,38 +0,0 @@ -apply from: rootProject.file('tools/gradle/commons/docker.gradle') - -task standardSourceTest { - ext { - imageName = '' - specPath = '' - configPath = '' - catalogPath = '' - } - - doFirst { - exec { - println('standard test inputs') - println("imageName: ${imageName}") - println("specPath: ${specPath}") - println("configPath: ${configPath}") - println("catalogPath: ${catalogPath}") - workingDir rootDir - commandLine 'docker', 'run', '--rm', '-i', - // so that it has access to docker - '-v', "/var/run/docker.sock:/var/run/docker.sock", - // when launching the container within a container, it mounts the directory from - // the host filesystem, not the parent container. this forces /tmp to be the - // same directory for host, parent container, and child container. - '-v', "/tmp:/tmp", - // mount the project dir. all provided input paths must be relative to that dir. - '-v', "${project.projectDir.absolutePath}:/test_input", - '--name', "std-test-${project.name}", 'airbyte/standard-source-test:dev', - '--imageName', imageName, - '--spec', "/test_input/${specPath}", - '--config', "/test_input/${configPath}", - '--catalog', "/test_input/${catalogPath}" - } - } -} -standardSourceTest.dependsOn(':airbyte-integrations:bases:standard-source-test:buildImage') -standardSourceTest.dependsOn(compileJava) -standardSourceTest.dependsOn(buildImage) diff --git a/tools/gradle/commons/integrations/test-image.gradle b/tools/gradle/commons/integrations/test-image.gradle deleted file mode 100644 index 5d1c73126cae..000000000000 --- a/tools/gradle/commons/integrations/test-image.gradle +++ /dev/null @@ -1,28 +0,0 @@ -// todo (cgardens) - dedupe this code with image.gradle. -import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage - -buildscript { - repositories { - jcenter() - } - dependencies { - classpath "com.bmuschko:gradle-docker-plugin:6.6.1" - } -} - -apply plugin: com.bmuschko.gradle.docker.DockerRemoteApiPlugin -apply from: rootProject.file('tools/gradle/commons/docker.gradle') - -task buildTestImage(type: DockerBuildImage) { - def testDockerfile = project.file('Dockerfile.test') - inputDir = projectDir - dockerFile = testDockerfile - images.add("${extractImageName(testDockerfile)}:dev") - - // Handles the case where there's an upstream dependency on a dev versioned image built by Gradle. - // The plugin doesn't recognize changes to the base image as a change that requires rebuilding. - // Docker will still cache the artifact, so it shouldn't be much slower. - outputs.upToDateWhen { false } -} - -buildImage.dependsOn(build)