diff --git a/.gitignore b/.gitignore index d6cafd854f0c..e7c2e09f6943 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,9 @@ build data .dockerversions +# Secrets secrets +!airbyte-integrations/connector-templates/**/secrets # Python *.egg-info diff --git a/airbyte-integrations/connector-templates/generator/plopfile.js b/airbyte-integrations/connector-templates/generator/plopfile.js index 4ec620f4a29e..b30419ad1d98 100644 --- a/airbyte-integrations/connector-templates/generator/plopfile.js +++ b/airbyte-integrations/connector-templates/generator/plopfile.js @@ -35,7 +35,7 @@ module.exports = function (plop) { console.log(getSuccessMessage(answers.name, plopApi.renderString(config.outputPath, answers), config.message)); }); - plop.setGenerator('Python Source', { + plop.setGenerator('source-python', { description: 'Generate an Airbyte Source written in Python', prompts: [{type: 'input', name: 'name', message: 'Source name, without the "source-" prefix e.g: "google-analytics"'}], actions: [ @@ -45,15 +45,8 @@ module.exports = function (plop) { destination: pythonSourceOutputRoot, base: pythonSourceInputRoot, templateFiles: `${pythonSourceInputRoot}/**/**`, - globOptions: {ignore:'.secrets'} }, // plop doesn't add dotfiles by default so we manually add them - { - type:'add', - abortOnFail: true, - templateFile: `${pythonSourceInputRoot}/.secrets/config.json.hbs`, - path: `${pythonSourceOutputRoot}/secrets/config.json` - }, { type:'add', abortOnFail: true, @@ -69,7 +62,7 @@ module.exports = function (plop) { {type: 'emitSuccess', outputPath: pythonSourceOutputRoot, message: "For a checklist of what to do next go to https://docs.airbyte.io/tutorials/building-a-python-source"}] }); - plop.setGenerator('Singer-based Python Source', { + plop.setGenerator('source-python-singer', { description: 'Generate an Airbyte Source written on top of a Singer Tap.', prompts: [{type: 'input', name: 'name', message: 'Source name, without the "source-" prefix e.g: "google-analytics"'}], actions: [ @@ -79,13 +72,6 @@ module.exports = function (plop) { destination: singerSourceOutputRoot, base: singerSourceInputRoot, templateFiles: `${singerSourceInputRoot}/**/**`, - globOptions: {ignore:'.secrets'} - }, - { - type:'add', - abortOnFail: true, - templateFile: `${singerSourceInputRoot}/.secrets/config.json.hbs`, - path: `${singerSourceOutputRoot}/secrets/config.json` }, { type:'add', @@ -103,7 +89,7 @@ module.exports = function (plop) { ] }); - plop.setGenerator('Generic Source', { + plop.setGenerator('source-generic', { description: 'Use if none of the other templates apply to your use case.', prompts: [{type: 'input', name: 'name', message: 'Source name, without the "source-" prefix e.g: "google-analytics"'}], actions: [ @@ -113,7 +99,6 @@ module.exports = function (plop) { destination: genericSourceOutputRoot, base: genericSourceInputRoot, templateFiles: `${genericSourceInputRoot}/**/**`, - globOptions: {ignore:'.secrets'} }, { type:'add', @@ -124,6 +109,4 @@ module.exports = function (plop) { {type: 'emitSuccess', outputPath: genericSourceOutputRoot} ] }); - - }; diff --git a/airbyte-integrations/connector-templates/source-python/.secrets/config.json.hbs b/airbyte-integrations/connector-templates/source-python/secrets/config.json.hbs similarity index 100% rename from airbyte-integrations/connector-templates/source-python/.secrets/config.json.hbs rename to airbyte-integrations/connector-templates/source-python/secrets/config.json.hbs diff --git a/airbyte-integrations/connector-templates/source-singer/.secrets/config.json.hbs b/airbyte-integrations/connector-templates/source-singer/secrets/config.json.hbs similarity index 100% rename from airbyte-integrations/connector-templates/source-singer/.secrets/config.json.hbs rename to airbyte-integrations/connector-templates/source-singer/secrets/config.json.hbs diff --git a/tools/integrations/manage.sh b/tools/integrations/manage.sh index 0b4d755354b1..c07bb6795b9c 100755 --- a/tools/integrations/manage.sh +++ b/tools/integrations/manage.sh @@ -4,26 +4,40 @@ set -e . tools/lib/lib.sh -_get_rule_base() { - local rule; rule=$(echo "$1" | tr -s / :) - echo ":$rule" -} +USAGE=" +Usage: $(basename "$0") +Available commands: + scaffold + build + publish +" _check_tag_exists() { DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$1" > /dev/null } +cmd_scaffold() { + echo "Scaffolding connector" + ( + cd airbyte-integrations/connector-templates/generator && + npm install && + npm run generate "$@" + ) +} + cmd_build() { - local path=$1 + local path=$1; shift || error "Missing target (root path of integration) $USAGE" + [ -d "$path" ] || error "Path must be the root path of the integration" echo "Building $path" - ./gradlew "$(_get_rule_base "$path"):clean" - ./gradlew "$(_get_rule_base "$path"):build" - ./gradlew "$(_get_rule_base "$path"):integrationTest" + ./gradlew "$(_to_gradle_path "$path" clean)" + ./gradlew "$(_to_gradle_path "$path" build)" + ./gradlew "$(_to_gradle_path "$path" integrationTest)" } cmd_publish() { - local path=$1 + local path=$1; shift || error "Missing target (root path of integration) $USAGE" + [ -d "$path" ] || error "Path must be the root path of the integration" cmd_build "$path" @@ -39,31 +53,20 @@ cmd_publish() { docker tag "$image_name:dev" "$versioned_image" docker tag "$image_name:dev" "$latest_image" - if _check_tag_exists $versioned_image; then + if _check_tag_exists "$versioned_image"; then error "You're trying to push a version that was already released ($versioned_image). Make sure you bump it up." fi echo "Publishing new version ($versioned_image)" - docker push $versioned_image - docker push $latest_image + docker push "$versioned_image" + docker push "$latest_image" } -USAGE=" - -Usage: $(basename $0) -" - main() { assert_root - local cmd=$1 - shift || error "Missing cmd $USAGE" - local path=$1 - shift || error "Missing target (root path of integration) $USAGE" - - [ -d "$path" ] || error "Path must be the root path of the integration" - - cmd_"$cmd" "$path" + local cmd=$1; shift || error "Missing cmd $USAGE" + cmd_"$cmd" "$@" } main "$@" diff --git a/tools/lib/lib.sh b/tools/lib/lib.sh index 605bed3df4a1..0ab96f8c6c7c 100644 --- a/tools/lib/lib.sh +++ b/tools/lib/lib.sh @@ -29,5 +29,12 @@ _get_docker_image_name() { _get_docker_label $1 io.airbyte.name } -VERSION=$(cat .env | grep "^VERSION=" | cut -d = -f 2) -SCRIPT_DIRECTORY=$(_script_directory) +_to_gradle_path() { + local path=$1 + local task=$2 + + echo ":$(echo "$path" | tr -s / :):${task}" +} + +VERSION=$(cat .env | grep "^VERSION=" | cut -d = -f 2); export VERSION +SCRIPT_DIRECTORY=$(_script_directory); export SCRIPT_DIRECTORY