From b955a5ac74f7759c83c3009e1aa83a3f35005d02 Mon Sep 17 00:00:00 2001 From: Michel Tricot Date: Sat, 17 Oct 2020 22:04:46 -0700 Subject: [PATCH] gitignore secrets at top level (#606) --- .gitignore | 2 ++ .../bigquery-destination/.gitignore | 1 - .../bigquery-destination/README.md | 21 +++++++++++++++++++ .../bigquery-destination/config/.gitkeep | 0 .../bigquery/BigQueryIntegrationTest.java | 2 +- .../bigquery/BigQueryDestinationTest.java | 2 +- .../singer/bigquery/destination/.gitignore | 1 - .../singer/bigquery/destination/README.md | 4 ++-- .../destinations/TestBigQueryDestination.java | 4 ++-- .../singer/stripe/source/.gitignore | 1 - .../singer/stripe/source/README.md | 4 ++-- .../sources/SingerStripeSourceTest.java | 2 +- .../stripe_abprotocol/source/.gitignore | 1 - .../singer/stripe_abprotocol/source/README.md | 4 ++-- .../sources/SingerStripeSourceTest.java | 2 +- tools/bin/ci_credentials.sh | 16 +++++++------- 16 files changed, 44 insertions(+), 23 deletions(-) delete mode 100644 airbyte-integrations/bigquery-destination/.gitignore create mode 100644 airbyte-integrations/bigquery-destination/README.md delete mode 100644 airbyte-integrations/bigquery-destination/config/.gitkeep delete mode 100644 airbyte-integrations/singer/bigquery/destination/.gitignore delete mode 100644 airbyte-integrations/singer/stripe/source/.gitignore delete mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/.gitignore diff --git a/.gitignore b/.gitignore index 3b729f527b65..6300868a0fd3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ build .DS_Store data +secrets + # Python *.egg-info __pycache__ diff --git a/airbyte-integrations/bigquery-destination/.gitignore b/airbyte-integrations/bigquery-destination/.gitignore deleted file mode 100644 index 04204c7c9d0e..000000000000 --- a/airbyte-integrations/bigquery-destination/.gitignore +++ /dev/null @@ -1 +0,0 @@ -config diff --git a/airbyte-integrations/bigquery-destination/README.md b/airbyte-integrations/bigquery-destination/README.md new file mode 100644 index 000000000000..90404739e98b --- /dev/null +++ b/airbyte-integrations/bigquery-destination/README.md @@ -0,0 +1,21 @@ +# BigQuery Test Configuration + +In order to test the BigQuery destination, you need a service account key file. + +## Community Contributor + +As a community contributor, you will need access to a GCP project and BigQuery to run tests. + +1. Go to the `Service Accounts` page on the GCP console +1. Click on `+ Create Service Account" button +1. Fill out a descriptive name/id/description +1. Click the edit icon next to the service account you created on the `IAM` page +1. Add the `BigQuery User` role +1. Go back to the `Service Accounts` page and use the actions modal to `Create Key` +1. Download this key as a JSON file +1. Move and rename this file to `secrets/credentials.json` + +## Airbyte Employee + +1. Access the `BigQuery Integration Test User` secret on Rippling under the `Engineering` folder +1. Create a file with the contents at `secrets/credentials.json` diff --git a/airbyte-integrations/bigquery-destination/config/.gitkeep b/airbyte-integrations/bigquery-destination/config/.gitkeep deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/airbyte-integrations/bigquery-destination/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryIntegrationTest.java b/airbyte-integrations/bigquery-destination/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryIntegrationTest.java index c49d74bf8ad6..4e7fe4faa386 100644 --- a/airbyte-integrations/bigquery-destination/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryIntegrationTest.java +++ b/airbyte-integrations/bigquery-destination/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryIntegrationTest.java @@ -54,7 +54,7 @@ public class BigQueryIntegrationTest extends TestDestination { private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryIntegrationTest.class); - private static final Path CREDENTIALS_PATH = Path.of("config/credentials.json"); + private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); private static final String CONFIG_DATASET_ID = "dataset_id"; private static final String CONFIG_PROJECT_ID = "project_id"; diff --git a/airbyte-integrations/bigquery-destination/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java b/airbyte-integrations/bigquery-destination/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java index fad718dbd739..09be924ee028 100644 --- a/airbyte-integrations/bigquery-destination/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java +++ b/airbyte-integrations/bigquery-destination/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java @@ -73,7 +73,7 @@ class BigQueryDestinationTest { - private static final Path CREDENTIALS_PATH = Path.of("config/credentials.json"); + private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDestinationTest.class); diff --git a/airbyte-integrations/singer/bigquery/destination/.gitignore b/airbyte-integrations/singer/bigquery/destination/.gitignore deleted file mode 100644 index dd04e98646d7..000000000000 --- a/airbyte-integrations/singer/bigquery/destination/.gitignore +++ /dev/null @@ -1 +0,0 @@ -config/* diff --git a/airbyte-integrations/singer/bigquery/destination/README.md b/airbyte-integrations/singer/bigquery/destination/README.md index 7464b607ca9a..90404739e98b 100644 --- a/airbyte-integrations/singer/bigquery/destination/README.md +++ b/airbyte-integrations/singer/bigquery/destination/README.md @@ -13,9 +13,9 @@ As a community contributor, you will need access to a GCP project and BigQuery t 1. Add the `BigQuery User` role 1. Go back to the `Service Accounts` page and use the actions modal to `Create Key` 1. Download this key as a JSON file -1. Move and rename this file to `config/credentials.json` +1. Move and rename this file to `secrets/credentials.json` ## Airbyte Employee 1. Access the `BigQuery Integration Test User` secret on Rippling under the `Engineering` folder -1. Create a file with the contents at `config/credentials.json` +1. Create a file with the contents at `secrets/credentials.json` diff --git a/airbyte-integrations/singer/bigquery/destination/src/test-integration/java/io/airbyte/integration_tests/destinations/TestBigQueryDestination.java b/airbyte-integrations/singer/bigquery/destination/src/test-integration/java/io/airbyte/integration_tests/destinations/TestBigQueryDestination.java index d0d1c0e46e50..8f8749397a60 100644 --- a/airbyte-integrations/singer/bigquery/destination/src/test-integration/java/io/airbyte/integration_tests/destinations/TestBigQueryDestination.java +++ b/airbyte-integrations/singer/bigquery/destination/src/test-integration/java/io/airbyte/integration_tests/destinations/TestBigQueryDestination.java @@ -68,7 +68,7 @@ class TestBigQueryDestination { - private static final String CONFIG_PATH = "config/credentials.json"; + private static final String CONFIG_PATH = "secrets/credentials.json"; private static final Logger LOGGER = LoggerFactory.getLogger(TestBigQueryDestination.class); @@ -191,7 +191,7 @@ private Process startTarget() throws IOException, WorkerException { } private void writeConfigFileToJobRoot() throws IOException { - String credentialsJsonString = new String(Files.readAllBytes(Paths.get("config/credentials.json"))); + String credentialsJsonString = new String(Files.readAllBytes(Paths.get("secrets/credentials.json"))); JsonNode credentials = Jsons.deserialize(credentialsJsonString); Map fullConfig = new HashMap<>(); diff --git a/airbyte-integrations/singer/stripe/source/.gitignore b/airbyte-integrations/singer/stripe/source/.gitignore deleted file mode 100644 index dd04e98646d7..000000000000 --- a/airbyte-integrations/singer/stripe/source/.gitignore +++ /dev/null @@ -1 +0,0 @@ -config/* diff --git a/airbyte-integrations/singer/stripe/source/README.md b/airbyte-integrations/singer/stripe/source/README.md index 8f9cf0bce67e..cd7fd8880e28 100644 --- a/airbyte-integrations/singer/stripe/source/README.md +++ b/airbyte-integrations/singer/stripe/source/README.md @@ -5,7 +5,7 @@ In order to test the Stripe source, you will need API credentials and the abilit ## Community Contributor 1. Create an empty account on Stripe. -1. Create a file at `config/config.json` with the following format using your client secret and account id: +1. Create a file at `secrets/config.json` with the following format using your client secret and account id: ``` { "client_secret": "sk_XXXXXXXXXXX", @@ -17,4 +17,4 @@ In order to test the Stripe source, you will need API credentials and the abilit ## Airbyte Employee 1. Access the `Stripe Integration Test Config` secret on Rippling under the `Engineering` folder -1. Create a file with the contents at `config/config.json` +1. Create a file with the contents at `secrets/config.json` diff --git a/airbyte-integrations/singer/stripe/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java b/airbyte-integrations/singer/stripe/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java index b3de786aabaf..11a852d70364 100644 --- a/airbyte-integrations/singer/stripe/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java +++ b/airbyte-integrations/singer/stripe/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java @@ -62,7 +62,7 @@ public class SingerStripeSourceTest { private static final String CATALOG = "catalog.json"; private static final String CONFIG = "config.json"; - private static final String CONFIG_PATH = "config/config.json"; + private static final String CONFIG_PATH = "secrets/config.json"; private static final String INVALID_CONFIG = "invalid_config.json"; protected Path jobRoot; diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/.gitignore b/airbyte-integrations/singer/stripe_abprotocol/source/.gitignore deleted file mode 100644 index dd04e98646d7..000000000000 --- a/airbyte-integrations/singer/stripe_abprotocol/source/.gitignore +++ /dev/null @@ -1 +0,0 @@ -config/* diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/README.md b/airbyte-integrations/singer/stripe_abprotocol/source/README.md index 12e117b92a9b..5681056cf6f7 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/README.md +++ b/airbyte-integrations/singer/stripe_abprotocol/source/README.md @@ -5,7 +5,7 @@ In order to test the Stripe source, you will need API credentials and the abilit ## Community Contributor 1. Create an empty account on Stripe. -1. Create a file at `config/config.json` with the following format using your client secret and account id: +1. Create a file at `secrets/config.json` with the following format using your client secret and account id: ``` { "client_secret": "sk_XXXXXXXXXXX", @@ -17,5 +17,5 @@ In order to test the Stripe source, you will need API credentials and the abilit ## Airbyte Employee 1. Access the `Stripe Integration Test Config` secret on Rippling under the `Engineering` folder -1. Create a file with the contents at `config/config.json` +1. Create a file with the contents at `secrets/config.json` diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java index 5c94ffb34394..a2fa723abaf7 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java @@ -65,7 +65,7 @@ public class SingerStripeSourceTest { private static final String CATALOG = "catalog.json"; private static final String CONFIG = "config.json"; - private static final String CONFIG_PATH = "config/config.json"; + private static final String CONFIG_PATH = "secrets/config.json"; private static final String INVALID_CONFIG = "invalid_config.json"; protected Path jobRoot; diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 19f3d92cde1f..6e9da507738b 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -2,12 +2,14 @@ set -e -mkdir airbyte-integrations/singer/bigquery/destination/config -echo "$BIGQUERY_INTEGRATION_TEST_CREDS" > airbyte-integrations/singer/bigquery/destination/config/credentials.json -echo "$BIGQUERY_INTEGRATION_TEST_CREDS" > airbyte-integrations/bigquery-destination/config/credentials.json +mkdir airbyte-integrations/singer/bigquery/destination/secrets +echo "$BIGQUERY_INTEGRATION_TEST_CREDS" > airbyte-integrations/singer/bigquery/destination/secrets/credentials.json -mkdir airbyte-integrations/singer/stripe/source/config -echo "$STRIPE_INTEGRATION_TEST_CREDS" > airbyte-integrations/singer/stripe/source/config/config.json +mkdir airbyte-integrations/bigquery-destination/secrets +echo "$BIGQUERY_INTEGRATION_TEST_CREDS" > airbyte-integrations/bigquery-destination/secrets/credentials.json -mkdir airbyte-integrations/singer/stripe_abprotocol/source/config -echo "$STRIPE_INTEGRATION_TEST_CREDS" > airbyte-integrations/singer/stripe_abprotocol/source/config/config.json +mkdir airbyte-integrations/singer/stripe/source/secrets +echo "$STRIPE_INTEGRATION_TEST_CREDS" > airbyte-integrations/singer/stripe/source/secrets/config.json + +mkdir airbyte-integrations/singer/stripe_abprotocol/source/secrets +echo "$STRIPE_INTEGRATION_TEST_CREDS" > airbyte-integrations/singer/stripe_abprotocol/source/secrets/config.json