Fix connector harness performance ci run (#31304)

Co-authored-by: xiaohansong <[email protected]>
airbytehq · Oct 13, 2023 · bc670da · bc670da
1 parent 1a90d6c
commit bc670da
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 261 deletions.
diff --git a/.github/workflows/connector-performance-command.yml b/.github/workflows/connector-performance-command.yml
@@ -4,7 +4,13 @@ on:
     inputs:
       connector:
         description: "Airbyte Connector"
+        type: choice
         required: true
+        options:
+          - connectors/source-postgres
+          - connectors/source-mysql
+          - connectors/destination-snowflake
+        default: "connectors/source-postgres"
       repo:
         description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos."
         required: false
@@ -102,6 +108,7 @@ jobs:
         with:
           repository: ${{ github.event.inputs.repo }}
           ref: ${{ github.event.inputs.gitref }}
+          fetch-depth: 0 # This is to fetch the main branch in case we are running on a different branch.
       - name: Install Java
         uses: actions/setup-java@v3
         with:
@@ -124,6 +131,7 @@ jobs:
           echo "harness_type=$the_harness" >> "$GITHUB_OUTPUT"
       - name: Write harness credentials
         run: |
+          export PATH="$PATH:/root/.local/bin"
           ci_credentials connectors-performance/$HARNESS_TYPE write-to-storage
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
@@ -144,6 +152,8 @@ jobs:
           connector_name=$(echo ${{ github.event.inputs.connector }} | cut -d / -f 2)
           echo "Running ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check"
           ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check
+        env:
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
       - name: KIND Kubernetes Cluster Setup
         uses: helm/[email protected]
         with:
@@ -180,6 +190,7 @@ jobs:
           kubectl logs --tail=1 $POD | while read line ; do line=${line#"$PREFIX"}; line=${line%"$SUFFIX"}; echo $line >> $GITHUB_OUTPUT ; done
           echo "$EOF" >> $GITHUB_OUTPUT
       - name: Link comment to workflow run
+        if: github.event.inputs.comment-id
         uses: peter-evans/create-or-update-comment@v2
         with:
           reactions: "+1"

diff --git a/...s-performance/source-harness/src/main/resources/catalogs/source-postgres/10m_catalog.json b/...s-performance/source-harness/src/main/resources/catalogs/source-postgres/10m_catalog.json
@@ -1,54 +1,9 @@
 {
   "streams": [
-    {
-      "stream": {
-        "name": "purchases",
-        "namespace": "10m_users",
-        "json_schema": {
-          "type": "object",
-          "properties": {
-            "id": {
-              "type": "number",
-              "airbyte_type": "integer"
-            },
-            "user_id": {
-              "type": "number",
-              "airbyte_type": "integer"
-            },
-            "product_id": {
-              "type": "number",
-              "airbyte_type": "integer"
-            },
-            "returned_at": {
-              "type": "string",
-              "format": "date-time",
-              "airbyte_type": "timestamp_with_timezone"
-            },
-            "purchased_at": {
-              "type": "string",
-              "format": "date-time",
-              "airbyte_type": "timestamp_with_timezone"
-            },
-            "added_to_cart_at": {
-              "type": "string",
-              "format": "date-time",
-              "airbyte_type": "timestamp_with_timezone"
-            }
-          }
-        },
-        "default_cursor_field": [],
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_primary_key": [["id"]]
-      },
-      "sync_mode": "full_refresh",
-      "primary_key": [["id"]],
-      "cursor_field": ["id"],
-      "destination_sync_mode": "append"
-    },
     {
       "stream": {
         "name": "users",
-        "namespace": "10m_users",
+        "namespace": "public",
         "json_schema": {
           "type": "object",
           "properties": {
@@ -111,51 +66,13 @@
         },
         "default_cursor_field": [],
         "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "full_refresh",
       "primary_key": [["id"]],
-      "cursor_field": ["updated_at"],
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "products",
-        "namespace": "10m_users",
-        "json_schema": {
-          "type": "object",
-          "properties": {
-            "id": {
-              "type": "number",
-              "airbyte_type": "integer"
-            },
-            "make": {
-              "type": "string"
-            },
-            "year": {
-              "type": "string"
-            },
-            "model": {
-              "type": "string"
-            },
-            "price": {
-              "type": "number"
-            },
-            "created_at": {
-              "type": "string",
-              "format": "date-time",
-              "airbyte_type": "timestamp_with_timezone"
-            }
-          }
-        },
-        "default_cursor_field": [],
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_primary_key": [["id"]]
-      },
-      "sync_mode": "full_refresh",
-      "primary_key": [["id"]],
-      "cursor_field": ["created_at"],
-      "destination_sync_mode": "append"
+      "cursor_field": [],
+      "destination_sync_mode": "overwrite"
     }
   ]
 }
diff --git a/...rs-performance/source-harness/src/main/resources/catalogs/source-postgres/1m_catalog.json b/...rs-performance/source-harness/src/main/resources/catalogs/source-postgres/1m_catalog.json
@@ -1,54 +1,9 @@
 {
   "streams": [
-    {
-      "stream": {
-        "name": "purchases",
-        "namespace": "1m_users",
-        "json_schema": {
-          "type": "object",
-          "properties": {
-            "id": {
-              "type": "number",
-              "airbyte_type": "integer"
-            },
-            "user_id": {
-              "type": "number",
-              "airbyte_type": "integer"
-            },
-            "product_id": {
-              "type": "number",
-              "airbyte_type": "integer"
-            },
-            "returned_at": {
-              "type": "string",
-              "format": "date-time",
-              "airbyte_type": "timestamp_with_timezone"
-            },
-            "purchased_at": {
-              "type": "string",
-              "format": "date-time",
-              "airbyte_type": "timestamp_with_timezone"
-            },
-            "added_to_cart_at": {
-              "type": "string",
-              "format": "date-time",
-              "airbyte_type": "timestamp_with_timezone"
-            }
-          }
-        },
-        "default_cursor_field": [],
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_primary_key": [["id"]]
-      },
-      "sync_mode": "full_refresh",
-      "primary_key": [["id"]],
-      "cursor_field": ["id"],
-      "destination_sync_mode": "append"
-    },
     {
       "stream": {
         "name": "users",
-        "namespace": "1m_users",
+        "namespace": "public",
         "json_schema": {
           "type": "object",
           "properties": {
@@ -111,51 +66,13 @@
         },
         "default_cursor_field": [],
         "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "full_refresh",
       "primary_key": [["id"]],
-      "cursor_field": ["updated_at"],
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "products",
-        "namespace": "1m_users",
-        "json_schema": {
-          "type": "object",
-          "properties": {
-            "id": {
-              "type": "number",
-              "airbyte_type": "integer"
-            },
-            "make": {
-              "type": "string"
-            },
-            "year": {
-              "type": "string"
-            },
-            "model": {
-              "type": "string"
-            },
-            "price": {
-              "type": "number"
-            },
-            "created_at": {
-              "type": "string",
-              "format": "date-time",
-              "airbyte_type": "timestamp_with_timezone"
-            }
-          }
-        },
-        "default_cursor_field": [],
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_primary_key": [["id"]]
-      },
-      "sync_mode": "full_refresh",
-      "primary_key": [["id"]],
-      "cursor_field": ["created_at"],
-      "destination_sync_mode": "append"
+      "cursor_field": [],
+      "destination_sync_mode": "overwrite"
     }
   ]
 }
diff --git a/...s-performance/source-harness/src/main/resources/catalogs/source-postgres/20m_catalog.json b/...s-performance/source-harness/src/main/resources/catalogs/source-postgres/20m_catalog.json
@@ -1,54 +1,9 @@
 {
   "streams": [
-    {
-      "stream": {
-        "name": "purchases",
-        "namespace": "20m_users",
-        "json_schema": {
-          "type": "object",
-          "properties": {
-            "id": {
-              "type": "number",
-              "airbyte_type": "integer"
-            },
-            "user_id": {
-              "type": "number",
-              "airbyte_type": "integer"
-            },
-            "product_id": {
-              "type": "number",
-              "airbyte_type": "integer"
-            },
-            "returned_at": {
-              "type": "string",
-              "format": "date-time",
-              "airbyte_type": "timestamp_with_timezone"
-            },
-            "purchased_at": {
-              "type": "string",
-              "format": "date-time",
-              "airbyte_type": "timestamp_with_timezone"
-            },
-            "added_to_cart_at": {
-              "type": "string",
-              "format": "date-time",
-              "airbyte_type": "timestamp_with_timezone"
-            }
-          }
-        },
-        "default_cursor_field": [],
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_primary_key": [["id"]]
-      },
-      "sync_mode": "full_refresh",
-      "primary_key": [["id"]],
-      "cursor_field": ["id"],
-      "destination_sync_mode": "append"
-    },
     {
       "stream": {
         "name": "users",
-        "namespace": "20m_users",
+        "namespace": "public",
         "json_schema": {
           "type": "object",
           "properties": {
@@ -111,51 +66,13 @@
         },
         "default_cursor_field": [],
         "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "full_refresh",
       "primary_key": [["id"]],
-      "cursor_field": ["updated_at"],
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "products",
-        "namespace": "20m_users",
-        "json_schema": {
-          "type": "object",
-          "properties": {
-            "id": {
-              "type": "number",
-              "airbyte_type": "integer"
-            },
-            "make": {
-              "type": "string"
-            },
-            "year": {
-              "type": "string"
-            },
-            "model": {
-              "type": "string"
-            },
-            "price": {
-              "type": "number"
-            },
-            "created_at": {
-              "type": "string",
-              "format": "date-time",
-              "airbyte_type": "timestamp_with_timezone"
-            }
-          }
-        },
-        "default_cursor_field": [],
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_primary_key": [["id"]]
-      },
-      "sync_mode": "full_refresh",
-      "primary_key": [["id"]],
-      "cursor_field": ["created_at"],
-      "destination_sync_mode": "append"
+      "cursor_field": [],
+      "destination_sync_mode": "overwrite"
     }
   ]
 }