-
Notifications
You must be signed in to change notification settings - Fork 4.2k
134 lines (126 loc) · 6.19 KB
/
airbyte-ci-tests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
name: Connector Ops CI - Pipeline Unit Test
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
on:
workflow_dispatch:
inputs:
airbyte_ci_binary_url:
description: "URL to airbyte-ci binary"
required: false
default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
pull_request:
types:
- opened
- reopened
- synchronize
jobs:
run-airbyte-ci-tests:
name: Run Airbyte CI tests
runs-on: "ci-runner-connector-test-large-dagger-0-6-4"
steps:
- name: Checkout Airbyte
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.ref }}
# IMPORTANT! This is nessesary to make sure that a status is reported on the PR
# even if the workflow is skipped. If we used github actions filters, the workflow
# would not be reported as skipped, but instead would be forever pending.
#
# I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE.
#
# Also it gets worse
#
# IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED.
# MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING.
- name: Get changed files
uses: tj-actions/changed-files@v39
id: changes
with:
files_yaml: |
ops:
- 'airbyte-ci/connectors/connector_ops/**'
- '!**/*.md'
base_images:
- 'airbyte-ci/connectors/connector_ops/**'
- 'airbyte-ci/connectors/base_images/**'
- '!**/*.md'
pipelines:
- 'airbyte-ci/connectors/connector_ops/**'
- 'airbyte-ci/connectors/base_images/**'
- 'airbyte-ci/connectors/pipelines/**'
- '!**/*.md'
metadata_lib:
- 'airbyte-ci/connectors/metadata_service/lib/**'
- '!**/*.md'
metadata_orchestrator:
- 'airbyte-ci/connectors/metadata_service/lib/**'
- 'airbyte-ci/connectors/metadata_service/orchestrator/**'
- '!**/*.md'
- name: Run airbyte-ci/connectors/connector_ops tests
if: steps.changes.outputs.ops_any_changed == 'true'
id: run-airbyte-ci-connectors-connector-ops-tests
uses: ./.github/actions/run-dagger-pipeline
with:
context: "pull_request"
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
subcommand: "test airbyte-ci/connectors/connector_ops"
airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
- name: Run airbyte-ci/connectors/pipelines tests
id: run-airbyte-ci-connectors-pipelines-tests
if: steps.changes.outputs.pipelines_any_changed == 'true'
uses: ./.github/actions/run-dagger-pipeline
with:
context: "pull_request"
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
subcommand: "test airbyte-ci/connectors/pipelines"
airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
- name: Run airbyte-ci/connectors/base_images tests
id: run-airbyte-ci-connectors-base-images-tests
if: steps.changes.outputs.base_images_any_changed == 'true'
uses: ./.github/actions/run-dagger-pipeline
with:
context: "pull_request"
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
subcommand: "test airbyte-ci/connectors/base_images"
airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
- name: Run test pipeline for the metadata lib
id: metadata_lib-test-pipeline
if: steps.changes.outputs.metadata_lib_any_changed == 'true'
uses: ./.github/actions/run-dagger-pipeline
with:
subcommand: "test airbyte-ci/connectors/metadata_service/lib/"
context: "pull_request"
github_token: ${{ secrets.GITHUB_TOKEN }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
- name: Run test for the metadata orchestrator
id: metadata_orchestrator-test-pipeline
if: steps.changes.outputs.metadata_orchestrator_any_changed == 'true'
uses: ./.github/actions/run-dagger-pipeline
with:
subcommand: "test airbyte-ci/connectors/metadata_service/orchestrator/"
context: "pull_request"
github_token: ${{ secrets.GITHUB_TOKEN }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}