diff --git a/.github/workflows/connector-performance-command.yml b/.github/workflows/connector-performance-command.yml index 5ed9bcf812d4..c86c798841f0 100644 --- a/.github/workflows/connector-performance-command.yml +++ b/.github/workflows/connector-performance-command.yml @@ -8,6 +8,35 @@ on: dataset: type: string required: true + repo: + description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." + type: string + required: false + default: "airbytehq/airbyte" + gitref: + description: "The git ref to check out from the specified repository." + type: string + required: false + default: master + uuid: + description: "Custom UUID of workflow run. Used because GitHub dispatches endpoint does not return workflow run id." + type: string + required: false + stream-number: + description: "Number of streams to use for destination performance measurement." + type: string + required: false + default: "1" + sync-mode: + description: "Sync mode to use for destination performance measurement." + required: false + type: string + default: "full_refresh" + report-to-datadog: + description: "Whether to report the performance test results to Datadog." + required: false + type: string + default: "true" workflow_dispatch: inputs: connector: @@ -50,7 +79,7 @@ on: - full_refresh - incremental default: "full_refresh" - reportToDatadog: + report-to-datadog: description: "Whether to report the performance test results to Datadog." required: false default: "false" @@ -60,7 +89,7 @@ jobs: timeout-minutes: 10 runs-on: ubuntu-latest steps: - - name: UUID ${{ github.event.inputs.uuid }} + - name: UUID ${{ inputs.uuid }} run: true start-test-runner: name: Start Build EC2 Runner @@ -74,8 +103,8 @@ jobs: - name: Checkout Airbyte uses: actions/checkout@v3 with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} + repository: ${{ inputs.repo }} + ref: ${{ inputs.gitref }} - name: Check PAT rate limits run: | ./tools/bin/find_non_rate_limited_PAT \ @@ -94,38 +123,37 @@ jobs: runs-on: ${{ needs.start-test-runner.outputs.label }} steps: - name: Link comment to workflow run - if: github.event.inputs.comment-id + if: inputs.comment-id uses: peter-evans/create-or-update-comment@v1 with: - comment-id: ${{ github.event.inputs.comment-id }} + comment-id: ${{ inputs.comment-id }} body: | #### Note: The following `dataset=` values are supported: `1m`(default), `10m`, `20m`, `bottleneck_stream1`, `bottleneck_stream_randomseed. For destinations only: you can also use `stream-numbers=N` to simulate N number of parallel streams. Additionally, `sync-mode=incremental` is supported for destinations. For example: `dataset=1m stream-numbers=2 sync-mode=incremental` - > :runner: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}. + > :runner: ${{inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}. - name: Search for valid connector name format id: regex uses: AsasInnab/regex-action@v1 with: regex_pattern: "^((connectors|bases)/)?[a-zA-Z0-9-_]+$" regex_flags: "i" # required to be set for this plugin - search_string: ${{ github.event.inputs.connector }} + search_string: ${{ inputs.connector }} - name: Validate input workflow format - if: steps.regex.outputs.first_match != github.event.inputs.connector + if: steps.regex.outputs.first_match != inputs.connector run: echo "The connector provided has an invalid format!" && exit 1 - name: Filter supported connectors - if: - "${{ github.event.inputs.connector != 'connectors/source-postgres' && - github.event.inputs.connector != 'connectors/source-mysql' && - github.event.inputs.connector != 'connectors/destination-snowflake' && + if: "${{ inputs.connector != 'connectors/source-postgres' && + inputs.connector != 'connectors/source-mysql' && + inputs.connector != 'connectors/destination-snowflake' && inputs.connector != 'connectors/source-mongodb-v2' }}" run: echo "Only connectors/source-postgres, source-mysql, source-mongodb-v2 and destination-snowflake currently supported by harness" && exit 1 - name: Checkout Airbyte uses: actions/checkout@v3 with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} + repository: ${{ inputs.repo }} + ref: ${{ inputs.gitref }} fetch-depth: 0 # This is to fetch the main branch in case we are running on a different branch. - name: Install Java uses: actions/setup-java@v3 @@ -145,7 +173,7 @@ jobs: - name: Source or Destination harness id: which-harness run: | - the_harness="$(echo ${{github.event.inputs.connector}} | sed 's/.*\///; s/-.*//')"-harness + the_harness="$(echo ${{inputs.connector}} | sed 's/.*\///; s/-.*//')"-harness echo "harness_type=$the_harness" >> "$GITHUB_OUTPUT" - name: Write harness credentials run: | @@ -165,9 +193,9 @@ jobs: - name: build connector shell: bash run: | - echo "Building... ${{github.event.inputs.connector}}" >> $GITHUB_STEP_SUMMARY + echo "Building... ${{inputs.connector}}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY # this is a blank line - connector_name=$(echo ${{ github.event.inputs.connector }} | cut -d / -f 2) + connector_name=$(echo ${{ inputs.connector }} | cut -d / -f 2) echo "Running ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check" ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check env: @@ -180,11 +208,11 @@ jobs: id: run-harness shell: bash env: - CONN: ${{ github.event.inputs.connector }} - DS: ${{ github.event.inputs.dataset }} - STREAM_NUMBER: ${{ github.event.inputs.stream-number }} - SYNC_MODE: ${{ github.event.inputs.sync-mode }} - REPORT_TO_DATADOG: ${{ inputs.reportToDatadog }} + CONN: ${{ inputs.connector }} + DS: ${{ inputs.dataset }} + STREAM_NUMBER: ${{ inputs.stream-number }} + SYNC_MODE: ${{ inputs.sync-mode }} + REPORT_TO_DATADOG: ${{ inputs.report-to-datadog }} PREFIX: '{"type":"LOG","log":{"level":"INFO","message":"INFO i.a.i.p.PerformanceTest(runTest):165' SUFFIX: '"}}' HARNESS_TYPE: ${{ steps.which-harness.outputs.harness_type }} @@ -208,11 +236,11 @@ jobs: kubectl logs --tail=1 $POD | while read line ; do line=${line#"$PREFIX"}; line=${line%"$SUFFIX"}; echo $line >> $GITHUB_OUTPUT ; done echo "$EOF" >> $GITHUB_OUTPUT - name: Link comment to workflow run - if: github.event.inputs.comment-id + if: inputs.comment-id uses: peter-evans/create-or-update-comment@v2 with: reactions: "+1" - comment-id: ${{ github.event.inputs.comment-id }} + comment-id: ${{ inputs.comment-id }} body: | ## Performance test Result: ``` diff --git a/.github/workflows/connector-performance-cron.yml b/.github/workflows/connector-performance-cron.yml index c8dd69d8405c..77748acbf755 100644 --- a/.github/workflows/connector-performance-cron.yml +++ b/.github/workflows/connector-performance-cron.yml @@ -1,34 +1,42 @@ name: Connector Performance Harness Cron -on: workflow_dispatch +on: + schedule: + # * is a special character in YAML so you have to quote this string + - # 5:30 and 17:30 + - cron: "30 5,17 * * *" + workflow_dispatch: # for manual triggers jobs: postgres-1m-run: uses: ./.github/workflows/connector-performance-command.yml with: - connector: connectors/source-postgres + connector: "connectors/source-postgres" dataset: 1m - postgres-10m-run: - uses: ./.github/workflows/connector-performance-command.yml - with: - connector: connectors/source-postgres - dataset: 10m - postgres-20m-run: + secrets: inherit + mysql-1m-run: uses: ./.github/workflows/connector-performance-command.yml with: - connector: connectors/source-postgres - dataset: 20m - mysql-1m-run: + connector: "connectors/source-mysql" + dataset: 1m + secrets: inherit + postgres-1m-run-incremental: uses: ./.github/workflows/connector-performance-command.yml with: - connector: connectors/source-mysql + connector: "connectors/source-postgres" dataset: 1m - mysql-10m-run: + sync-mode: "incremental" + secrets: inherit + mysql-1m-run-incremental: uses: ./.github/workflows/connector-performance-command.yml with: - connector: connectors/source-mysql - dataset: 10m - mysql-20m-run: + connector: "connectors/source-mysql" + dataset: 1m + sync-mode: "incremental" + secrets: inherit + mongodb-1m-run-incremental: uses: ./.github/workflows/connector-performance-command.yml with: - connector: connectors/source-mysql - dataset: 20m + connector: "connectors/source-mongodb-v2" + dataset: 1m + sync-mode: "incremental" + secrets: inherit