-
Notifications
You must be signed in to change notification settings - Fork 4.3k
128 lines (123 loc) · 6.06 KB
/
publish_connectors.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
name: Connector Ops CI - Publish Connectors
on:
push:
branches:
- master
paths:
- "airbyte-integrations/connectors/**/metadata.yaml"
workflow_dispatch:
inputs:
connectors-options:
description: "Options to pass to the 'airbyte-ci connectors' command group."
default: "--name=source-pokeapi"
publish-options:
description: "Options to pass to the 'airbyte-ci connectors publish' command. Use --pre-release or --main-release depending on whether you want to publish a dev image or not. "
default: "--pre-release"
runs-on:
type: string
default: ci-runner-connector-publish-large-dagger-0-9-5
required: true
airbyte-ci-binary-url:
description: "URL to airbyte-ci binary"
required: false
default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
jobs:
publish_connectors:
name: Publish connectors
runs-on: ${{ inputs.runs-on || 'ci-runner-connector-publish-large-dagger-0-9-5' }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v3
- name: Publish modified connectors [On merge to master]
id: publish-modified-connectors
if: github.event_name == 'push'
uses: ./.github/actions/run-dagger-pipeline
with:
context: "master"
dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
github_token: ${{ secrets.GITHUB_TOKEN }}
metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}
spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release"
- name: Publish connectors [manual]
id: publish-connectors
if: github.event_name == 'workflow_dispatch'
uses: ./.github/actions/run-dagger-pipeline
with:
context: "manual"
dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
github_token: ${{ secrets.GITHUB_TOKEN }}
metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}
spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}"
airbyte_ci_binary_url: ${{ github.event.inputs.airbyte-ci-binary-url }}
set-instatus-incident-on-failure:
name: Create Instatus Incident on Failure
runs-on: ubuntu-latest
needs:
- publish_connectors
if: ${{ failure() && github.ref == 'refs/heads/master' }}
steps:
- name: Call Instatus Webhook
uses: joelwmale/webhook-action@master
with:
url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }}
body: '{ "trigger": "down", "status": "HASISSUES" }'
set-instatus-incident-on-success:
name: Create Instatus Incident on Success
runs-on: ubuntu-latest
needs:
- publish_connectors
if: ${{ success() && github.ref == 'refs/heads/master' }}
steps:
- name: Call Instatus Webhook
uses: joelwmale/webhook-action@master
with:
url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }}
body: '{ "trigger": "up" }'
notify-failure-slack-channel:
name: "Notify Slack Channel on Build Failures"
runs-on: ubuntu-latest
needs:
- publish_connectors
if: ${{ failure() && github.ref == 'refs/heads/master' }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v3
- name: Match GitHub User to Slack User
id: match-github-to-slack-user
uses: ./.github/actions/match-github-to-slack-user
env:
AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }}
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Publish to OSS Build Failure Slack Channel
uses: abinoda/slack-action@master
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
with:
args: >-
{\"channel\":\"C056HGD1QSW\", \"blocks\":[
{\"type\":\"divider\"},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Publish Connector Failed! :bangbang: \n\n\"}},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}},
{\"type\":\"divider\"}]}