Skip to content

Connector Ops CI - Publish Connectors #1666

Connector Ops CI - Publish Connectors

Connector Ops CI - Publish Connectors #1666

name: Connector Ops CI - Publish Connectors
on:
push:
branches:
- master
paths:
- "airbyte-integrations/connectors/**/metadata.yaml"
workflow_dispatch:
inputs:
connectors-options:
description: "Options to pass to the 'airbyte-ci connectors' command group."
default: "--name=source-pokeapi"
publish-options:
description: "Options to pass to the 'airbyte-ci connectors publish' command. Use --pre-release or --main-release depending on whether you want to publish a dev image or not. "
default: "--pre-release"
runs-on:
type: string
default: conn-prod-xlarge-runner
required: true
jobs:
publish_connectors:
name: Publish connectors
runs-on: ${{ inputs.runs-on || 'conn-prod-xlarge-runner' }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v3
- name: Publish modified connectors [On merge to master]
id: publish-modified-connectors
if: github.event_name == 'push'
uses: ./.github/actions/run-dagger-pipeline
with:
context: "master"
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
github_token: ${{ secrets.GITHUB_TOKEN }}
metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}
spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release"
- name: Publish connectors [manual]
id: publish-connectors
if: github.event_name == 'workflow_dispatch'
uses: ./.github/actions/run-dagger-pipeline
with:
context: "manual"
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
github_token: ${{ secrets.GITHUB_TOKEN }}
metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}
spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}"
set-instatus-incident-on-failure:
name: Create Instatus Incident on Failure
runs-on: ubuntu-latest
needs:
- publish_connectors
if: ${{ failure() && github.ref == 'refs/heads/master' }}
steps:
- name: Call Instatus Webhook
uses: joelwmale/webhook-action@master
with:
url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }}
body: '{ "trigger": "down", "status": "HASISSUES" }'
set-instatus-incident-on-success:
name: Create Instatus Incident on Success
runs-on: ubuntu-latest
needs:
- publish_connectors
if: ${{ success() && github.ref == 'refs/heads/master' }}
steps:
- name: Call Instatus Webhook
uses: joelwmale/webhook-action@master
with:
url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }}
body: '{ "trigger": "up" }'
notify-failure-slack-channel:
name: "Notify Slack Channel on Build Failures"
runs-on: ubuntu-latest
needs:
- publish_connectors
if: ${{ failure() && github.ref == 'refs/heads/master' }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v3
- name: Match GitHub User to Slack User
id: match-github-to-slack-user
uses: ./.github/actions/match-github-to-slack-user
env:
AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }}
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Publish to OSS Build Failure Slack Channel
uses: abinoda/slack-action@master
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
with:
args: >-
{\"channel\":\"C056HGD1QSW\", \"blocks\":[
{\"type\":\"divider\"},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Publish Connector Failed! :bangbang: \n\n\"}},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}},
{\"type\":\"divider\"}]}