Hide connector scaffolding behind a script (airbytehq#2080)

.gitignore

@@ -6,7 +6,9 @@ build
 data
 .dockerversions
 
+# Secrets
 secrets
+!airbyte-integrations/connector-templates/**/secrets
 
 # Python
 *.egg-info

airbyte-integrations/connector-templates/generator/plopfile.js

@@ -35,7 +35,7 @@ module.exports = function (plop) {
       console.log(getSuccessMessage(answers.name, plopApi.renderString(config.outputPath, answers), config.message));
   });
 
-  plop.setGenerator('Python Source', {
+  plop.setGenerator('source-python', {
     description: 'Generate an Airbyte Source written in Python',
     prompts: [{type: 'input', name: 'name', message: 'Source name, without the "source-" prefix e.g: "google-analytics"'}],
     actions: [
@@ -45,15 +45,8 @@ module.exports = function (plop) {
         destination: pythonSourceOutputRoot,
         base: pythonSourceInputRoot,
         templateFiles: `${pythonSourceInputRoot}/**/**`,
-        globOptions: {ignore:'.secrets'}
       },
         // plop doesn't add dotfiles by default so we manually add them
-      {
-        type:'add',
-        abortOnFail: true,
-        templateFile: `${pythonSourceInputRoot}/.secrets/config.json.hbs`,
-        path: `${pythonSourceOutputRoot}/secrets/config.json`
-      },
       {
         type:'add',
         abortOnFail: true,
@@ -69,7 +62,7 @@ module.exports = function (plop) {
       {type: 'emitSuccess', outputPath: pythonSourceOutputRoot, message: "For a checklist of what to do next go to https://docs.airbyte.io/tutorials/building-a-python-source"}]
   });
 
-  plop.setGenerator('Singer-based Python Source', {
+  plop.setGenerator('source-python-singer', {
     description: 'Generate an Airbyte Source written on top of a Singer Tap.',
     prompts: [{type: 'input', name: 'name', message: 'Source name, without the "source-" prefix e.g: "google-analytics"'}],
     actions: [
@@ -79,13 +72,6 @@ module.exports = function (plop) {
         destination: singerSourceOutputRoot,
         base: singerSourceInputRoot,
         templateFiles: `${singerSourceInputRoot}/**/**`,
-        globOptions: {ignore:'.secrets'}
-      },
-      {
-        type:'add',
-        abortOnFail: true,
-        templateFile: `${singerSourceInputRoot}/.secrets/config.json.hbs`,
-        path: `${singerSourceOutputRoot}/secrets/config.json`
       },
       {
         type:'add',
@@ -103,7 +89,7 @@ module.exports = function (plop) {
     ]
   });
 
-  plop.setGenerator('Generic Source', {
+  plop.setGenerator('source-generic', {
       description: 'Use if none of the other templates apply to your use case.',
       prompts: [{type: 'input', name: 'name', message: 'Source name, without the "source-" prefix e.g: "google-analytics"'}],
       actions: [
@@ -113,7 +99,6 @@ module.exports = function (plop) {
           destination: genericSourceOutputRoot,
           base: genericSourceInputRoot,
           templateFiles: `${genericSourceInputRoot}/**/**`,
-          globOptions: {ignore:'.secrets'}
         },
         {
           type:'add',
@@ -124,6 +109,4 @@ module.exports = function (plop) {
           {type: 'emitSuccess', outputPath: genericSourceOutputRoot}
       ]
     });
-
-
 };

tools/integrations/manage.sh

@@ -4,26 +4,40 @@ set -e
 
 . tools/lib/lib.sh
 
-_get_rule_base() {
-  local rule; rule=$(echo "$1" | tr -s / :)
-  echo ":$rule"
-}
+USAGE="
+Usage: $(basename "$0") <cmd>
+Available commands:
+  scaffold
+  build  <integration_root_path>
+  publish  <integration_root_path>
+"
 
 _check_tag_exists() {
   DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$1" > /dev/null
 }
 
+cmd_scaffold() {
+  echo "Scaffolding connector"
+  (
+    cd airbyte-integrations/connector-templates/generator &&
+    npm install &&
+    npm run generate "$@"
+  )
+}
+
 cmd_build() {
-  local path=$1
+  local path=$1; shift || error "Missing target (root path of integration) $USAGE"
+  [ -d "$path" ] || error "Path must be the root path of the integration"
 
   echo "Building $path"
-  ./gradlew "$(_get_rule_base "$path"):clean"
-  ./gradlew "$(_get_rule_base "$path"):build"
-  ./gradlew "$(_get_rule_base "$path"):integrationTest"
+  ./gradlew "$(_to_gradle_path "$path" clean)"
+  ./gradlew "$(_to_gradle_path "$path" build)"
+  ./gradlew "$(_to_gradle_path "$path" integrationTest)"
 }
 
 cmd_publish() {
-  local path=$1
+  local path=$1; shift || error "Missing target (root path of integration) $USAGE"
+  [ -d "$path" ] || error "Path must be the root path of the integration"
 
   cmd_build "$path"
 
@@ -39,31 +53,20 @@ cmd_publish() {
   docker tag "$image_name:dev" "$versioned_image"
   docker tag "$image_name:dev" "$latest_image"
 
-  if _check_tag_exists $versioned_image; then
+  if _check_tag_exists "$versioned_image"; then
     error "You're trying to push a version that was already released ($versioned_image). Make sure you bump it up."
   fi
 
   echo "Publishing new version ($versioned_image)"
-  docker push $versioned_image
-  docker push $latest_image
+  docker push "$versioned_image"
+  docker push "$latest_image"
 }
 
-USAGE="
-
-Usage: $(basename $0) <build|publish> <integration_root_path>
-"
-
 main() {
   assert_root
 
-  local cmd=$1
-  shift || error "Missing cmd $USAGE"
-  local path=$1
-  shift || error "Missing target (root path of integration) $USAGE"
-
-  [ -d "$path" ] || error "Path must be the root path of the integration"
-
-  cmd_"$cmd" "$path"
+  local cmd=$1; shift || error "Missing cmd $USAGE"
+  cmd_"$cmd" "$@"
 }
 
 main "$@"

tools/lib/lib.sh

@@ -29,5 +29,12 @@ _get_docker_image_name() {
   _get_docker_label $1 io.airbyte.name
 }
 
-VERSION=$(cat .env | grep "^VERSION=" | cut -d = -f 2)
-SCRIPT_DIRECTORY=$(_script_directory)
+_to_gradle_path() {
+  local path=$1
+  local task=$2
+
+  echo ":$(echo "$path" | tr -s / :):${task}"
+}
+
+VERSION=$(cat .env | grep "^VERSION=" | cut -d = -f 2); export VERSION
+SCRIPT_DIRECTORY=$(_script_directory); export SCRIPT_DIRECTORY