airbyte-ci-tests.yml

name: Connector Ops CI - Pipeline Unit Test

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  workflow_dispatch:
  pull_request:
    types:
      - opened
      - reopened
      - synchronize
jobs:
  get_ci_runner:
    runs-on: ubuntu-latest
    name: Get CI runner
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
        with:
          ref: ${{ github.head_ref }}
          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
          fetch-depth: 1
      - name: Get CI runner
        id: get_ci_runner
        uses: ./.github/actions/airbyte-ci-requirements
        with:
          runner_type: "test"
          runner_size: "large"
          airbyte_ci_command: "test"
          github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
    outputs:
      runner_name: ${{ steps.get_ci_runner.outputs.runner_name }}
  run-airbyte-ci-tests:
    name: Run Airbyte CI tests
    needs: get_ci_runner
    runs-on: ${{ needs.get_ci_runner.outputs.runner_name }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.ref }}

      # IMPORTANT! This is nessesary to make sure that a status is reported on the PR
      # even if the workflow is skipped. If we used github actions filters, the workflow
      # would not be reported as skipped, but instead would be forever pending.
      #
      # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE.
      #
      # Also it gets worse
      #
      # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED.
      # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING.
      - name: Get changed files
        uses: tj-actions/changed-files@v39
        id: changes
        with:
          files_yaml: |
            ops:
              - 'airbyte-ci/connectors/connector_ops/**'
              - '!**/*.md'
            base_images:
              - 'airbyte-ci/connectors/connector_ops/**'
              - 'airbyte-ci/connectors/base_images/**'
              - '!**/*.md'
            pipelines:
              - 'airbyte-ci/connectors/connector_ops/**'
              - 'airbyte-ci/connectors/base_images/**'
              - 'airbyte-ci/connectors/pipelines/**'
              - '!**/*.md'
            metadata_lib:
              - 'airbyte-ci/connectors/metadata_service/lib/**'
              - '!**/*.md'
            metadata_orchestrator:
              - 'airbyte-ci/connectors/metadata_service/lib/**'
              - 'airbyte-ci/connectors/metadata_service/orchestrator/**'
              - '!**/*.md'
            airbyte_lib:
              - 'airbyte_lib/**'
              - '!**/*.md'

      - name: Run airbyte-ci/connectors/connector_ops tests
        #if: steps.changes.outputs.ops_any_changed == 'true'
        id: run-airbyte-ci-connectors-connector-ops-tests
        uses: ./.github/actions/run-dagger-pipeline
        with:
          context: "pull_request"
          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
          subcommand: "test airbyte-ci/connectors/connector_ops --poetry-run-command='pytest tests'"
          tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}

      - name: Run airbyte-ci/connectors/pipelines tests
        id: run-airbyte-ci-connectors-pipelines-tests
        #if: steps.changes.outputs.pipelines_any_changed == 'true'
        uses: ./.github/actions/run-dagger-pipeline
        with:
          context: "pull_request"
          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
          subcommand: "test airbyte-ci/connectors/pipelines --poetry-run-command='pytest tests' --poetry-run-command='mypy pipelines --disallow-untyped-defs' --poetry-run-command='ruff check pipelines'"
          tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}

      - name: Run airbyte-ci/connectors/base_images tests
        id: run-airbyte-ci-connectors-base-images-tests
        #if: steps.changes.outputs.base_images_any_changed == 'true'
        uses: ./.github/actions/run-dagger-pipeline
        with:
          context: "pull_request"
          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
          subcommand: "test airbyte-ci/connectors/base_images --poetry-run-command='pytest tests'"
          tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}

      - name: Run test pipeline for the metadata lib
        id: metadata_lib-test-pipeline
        #if: steps.changes.outputs.metadata_lib_any_changed == 'true'
        uses: ./.github/actions/run-dagger-pipeline
        with:
          subcommand: "test airbyte-ci/connectors/metadata_service/lib/ --poetry-run-command='pytest tests'"
          context: "pull_request"
          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
          github_token: ${{ secrets.GITHUB_TOKEN }}
          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
          tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}

      - name: Run test for the metadata orchestrator
        id: metadata_orchestrator-test-pipeline
        #if: steps.changes.outputs.metadata_orchestrator_any_changed == 'true'
        uses: ./.github/actions/run-dagger-pipeline
        with:
          subcommand: "test airbyte-ci/connectors/metadata_service/orchestrator/ --poetry-run-command='pytest tests'"
          context: "pull_request"
          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
          github_token: ${{ secrets.GITHUB_TOKEN }}
          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
          tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}

      - name: Run airbyte-lib tests
        #if: steps.changes.outputs.airbyte_lib_any_changed == 'true'
        id: run-airbyte-lib-tests
        uses: ./.github/actions/run-dagger-pipeline
        with:
          context: "pull_request"
          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
          subcommand: "test airbyte-lib --side-car-docker-engine --pass-env-var=GCP_GSM_CREDENTIALS --poetry-run-command='pytest'"
          tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}