forked from apache/airflow
-
Notifications
You must be signed in to change notification settings - Fork 0
154 lines (150 loc) · 8.12 KB
/
ci-teradata-main-systemtest-dashboard.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: airflow-teradata-main-systemtest
on:
push:
branches:
- main
schedule:
- cron: "0 6 * * *"
workflow_dispatch:
permissions:
contents: write
pages: write
id-token: write
jobs:
teradata-systemtest:
name: "Manage Teradata Provider System Tests Dashboard"
runs-on: "ubuntu-22.04"
steps:
- name: Show current user
run: |
whoami
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout Teradata Airflow Source Code"
uses: actions/checkout@v4
with:
ref: main
persist-credentials: false
- name: "Github Event and Branch Details"
run: echo "The current checked out branch is main. Workflow trigged Event- ${{ github.event_name }}. The fully qualified reference (ref) branch that triggered the workflow run- ${{ github.ref }}"
- name: "Install Python"
uses: actions/setup-python@v4
with:
python-version: 3.x
cache: 'pip'
- name: "Checkout ClearScape Scripts and Report Generation supported files"
run: |
git fetch
git checkout origin/${{vars.WORKFLOW_HELP_BRANCH}} -- .github/workflows/scripts/createTestEnvironments.sh
git checkout origin/${{vars.WORKFLOW_HELP_BRANCH}} -- .github/workflows/scripts/deleteTestEnvironments.sh
git checkout origin/${{vars.WORKFLOW_HELP_BRANCH}} -- .github/workflows/system-tests-files/report_generate_index_html.py
git checkout origin/${{vars.WORKFLOW_HELP_BRANCH}} -- .github/workflows/system-tests-files/report_index_template.html
git checkout origin/${{vars.WORKFLOW_HELP_BRANCH}} -- .github/workflows/system-tests-files/report_parse_xml.py
cp .github/workflows/system-tests-files/report_generate_index_html.py .
cp .github/workflows/system-tests-files/report_index_template.html .
cp .github/workflows/system-tests-files/report_parse_xml.py .
- name: "Download System Test Log Report"
uses: dawidd6/action-download-artifact@v3
with:
workflow: ci-teradata-main-systemtest-dashboard.yml
workflow_conclusion: completed
name: airflow-system-test-report
path: .
- name: "Display content of report csv"
run: |
cat reporttest.csv
- name: "Define CSAE env name"
id: define-environment-name
run: |
echo "env-name=airflow-ci-$(date +%s%N)" >> $GITHUB_OUTPUT
- name: "Create CSAE environment"
id: create-csae-environment
shell: bash
run: |
echo Clearscape Environment name : ${{ steps.define-environment-name.outputs.env-name}}
chmod 777 .github/workflows/scripts/createTestEnvironments.sh
.github/workflows/scripts/createTestEnvironments.sh
env:
CSAE_TOKEN: ${{ secrets.CSAE_AIRFLOW_TOKEN }}
CSAE_ENV_PASSWORD: ${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}
CSAE_ENV_NAME: ${{ steps.define-environment-name.outputs.env-name}}
- name: "Install Airflow Teradata Provider and Run System tests"
run: |
echo Clearscape Environment name : ${{ steps.create-csae-environment.outputs.teradata-server-name }}
sudo apt update
sudo apt-get install -y apt-transport-https apt-utils build-essential ca-certificates dirmngr \
freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4 libev-dev libffi-dev libgeos-dev \
libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev libsasl2-modules \
libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client pkgconf sasl2-bin \
software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev python3-venv python3-pip pipx
pip install pytest time-machine pytest-asyncio pytest-md-report cryptography pytest-html tailer pandas jinja2 sqlalchemy hatch
pip install -e ".[devel,teradata, microsoft.azure, amazon]"
export SYSTEM_TESTS_ENV_ID=teradatasystemtest
export AIRFLOW_CONN_TERADATA_DEFAULT='teradata://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}'
export AIRFLOW_CONN_TERADATA_LAKE='teradata://${{ vars.LAKE_ENV_USERNAME }}:${{ secrets.LAKE_ENV_PASSWORD }}@${{ vars.LAKE_SERVER_NAME }}:/${{ vars.LAKE_ENV_USERNAME }}'
export AIRFLOW_CONN_TERADATA_SSL_DEFAULT='teradata://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}?sslmode=allow'
export AIRFLOW_CONN_TERADATA_SP_CALL='teradata://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}?tmode=TERA'
export AIRFLOW_CONN_WASB_DEFAULT='wasb://${{ vars.AZURE_BLOB_ACCOUNTNAME }}:${{ secrets.AZURE_BLOB_ACCOUNT_SECRET_KEY_URI }}@'
export AIRFLOW_CONN_AWS_DEFAULT=aws://${{ vars.AWS_ACCESS_KEY_ID }}:${{ secrets.AWS_SECRET_ACCESS_KEY }}@/?aws_session_token=${{ secrets.AWS_SESSION_TOKEN }}
export AIRFLOW_VAR_AZURE_BLOB_ACCOUNTNAME=${{ vars.AZURE_BLOB_ACCOUNTNAME }}
export AIRFLOW_VAR_AZURE_BLOB_ACCOUNT_SECRET_KEY=${{ secrets.AZURE_BLOB_ACCOUNT_SECRET_KEY }}
export AIRFLOW_VAR_AWS_ACCESS_KEY_ID=${{ secrets.TERADATA_DOWNLOAD_AWS_SECRET_ACCESS_ID }}
export AIRFLOW_VAR_AWS_SECRET_ACCESS_KEY=${{ secrets.TERADATA_DOWNLOAD_AWS_SECRET_ACCESS_KEY }}
pytest --system --junitxml=report_test.xml providers/tests/system/teradata/
cat report_test.xml
- name: "Delete CSAE environment"
if: always()
run: |
chmod 777 .github/workflows/scripts/deleteTestEnvironments.sh
.github/workflows/scripts/deleteTestEnvironments.sh
env:
CSAE_TOKEN: ${{ secrets.CSAE_AIRFLOW_TOKEN }}
CSAE_ENV_NAME: ${{ steps.define-environment-name.outputs.env-name}}
- name: "Generate Teradata Airflow Provider System Tests Report"
if: always()
run: |
cat report_test.xml
python report_parse_xml.py
python report_generate_index_html.py
mkdir teradata_report
cp index.html teradata_report/index.html
cat reporttest.csv
- name: Upload airflow teradata system test report
if: always()
uses: actions/upload-artifact@v4
with:
name: airflow-system-test-report
path: |
reporttest.csv
report_test.xml
- name: "Build GITHUB Page for Teradata Airflow Provider System Tests Report"
if: always()
uses: actions/jekyll-build-pages@v1
with:
source: ./teradata_report
destination: ./_site
- name: "Upload Teradata Airflow Provider System Tests Report to GITHUB page"
if: always()
uses: actions/upload-pages-artifact@v2
- name: "Publish Teradata Airflow Provider System Tests Report to GITHUB page"
if: always()
id: deployment
uses: actions/deploy-pages@v2