automatic update of anyway-master-commit.txt #517
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| jobs: | |
| ci: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - env: | |
| ETL_IMAGE_NAME: "ghcr.io/data-for-change/anyway-etl/anyway-etl" | |
| NGINX_IMAGE_NAME: "ghcr.io/data-for-change/anyway-etl/anyway-etl-nginx" | |
| AIRFLOW_IMAGE_NAME: "ghcr.io/data-for-change/anyway-etl/anyway-etl-airflow" | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| DFC_K8S_DEPLOY_KEY: ${{ secrets.DFC_K8S_DEPLOY_KEY }} | |
| run: | | |
| if [[ "${GITHUB_REF}" =~ ^refs/tags/.* ]]; then | |
| TAG_NAME="${GITHUB_REF/refs\/tags\//}" | |
| else | |
| TAG_NAME="" | |
| fi | |
| echo "${GITHUB_TOKEN}" | docker login ghcr.io -u data-for-change --password-stdin &&\ | |
| if docker pull "${ETL_IMAGE_NAME}:latest"; then | |
| CACHE_FROM_ARG="--cache-from ${ETL_IMAGE_NAME}:latest" | |
| else | |
| CACHE_FROM_ARG="" | |
| fi &&\ | |
| docker build $CACHE_FROM_ARG -t etl . &&\ | |
| docker tag etl "${ETL_IMAGE_NAME}:${GITHUB_SHA}" &&\ | |
| docker push "${ETL_IMAGE_NAME}:${GITHUB_SHA}" &&\ | |
| if docker pull "${AIRFLOW_IMAGE_NAME}:latest"; then | |
| CACHE_FROM_ARG="--cache-from ${AIRFLOW_IMAGE_NAME}:latest" | |
| else | |
| CACHE_FROM_ARG="" | |
| fi &&\ | |
| docker build $CACHE_FROM_ARG -t airflow airflow_server &&\ | |
| docker tag airflow "${AIRFLOW_IMAGE_NAME}:${GITHUB_SHA}" &&\ | |
| docker push "${AIRFLOW_IMAGE_NAME}:${GITHUB_SHA}" &&\ | |
| if docker pull "${NGINX_IMAGE_NAME}:latest"; then | |
| CACHE_FROM_ARG="--cache-from ${NGINX_IMAGE_NAME}:latest" | |
| else | |
| CACHE_FROM_ARG="" | |
| fi &&\ | |
| docker build $CACHE_FROM_ARG -t aenginx nginx &&\ | |
| docker tag aenginx "${NGINX_IMAGE_NAME}:${GITHUB_SHA}" &&\ | |
| docker push "${NGINX_IMAGE_NAME}:${GITHUB_SHA}" &&\ | |
| if [ "${GITHUB_REF}" == "refs/heads/main" ]; then | |
| PUSH_TAG=latest | |
| elif [ "${TAG_NAME}" != "" ]; then | |
| PUSH_TAG="${TAG_NAME}" | |
| fi &&\ | |
| if [ "${PUSH_TAG}" != "" ]; then | |
| docker tag etl "${ETL_IMAGE_NAME}:${PUSH_TAG}" &&\ | |
| docker push "${ETL_IMAGE_NAME}:${PUSH_TAG}" &&\ | |
| docker tag airflow "${AIRFLOW_IMAGE_NAME}:${PUSH_TAG}" &&\ | |
| docker push "${AIRFLOW_IMAGE_NAME}:${PUSH_TAG}" &&\ | |
| docker tag aenginx "${NGINX_IMAGE_NAME}:${PUSH_TAG}" &&\ | |
| docker push "${NGINX_IMAGE_NAME}:${PUSH_TAG}" &&\ | |
| if [ "${TAG_NAME}" != "" ] || bin/ci_check_allow_deploy.sh; then | |
| cd `mktemp -d` &&\ | |
| echo "${DFC_K8S_DEPLOY_KEY}" > dfc_k8s_deploy_key &&\ | |
| chmod 400 dfc_k8s_deploy_key &&\ | |
| export GIT_SSH_COMMAND="ssh -i `pwd`/dfc_k8s_deploy_key -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" &&\ | |
| git clone [email protected]:data-for-change/dfc-k8s.git &&\ | |
| cd dfc-k8s &&\ | |
| git config --global user.name "Anyway ETL CI" &&\ | |
| git config --global user.email "anyway-etl-ci@localhost" &&\ | |
| if [ "${TAG_NAME}" != "" ]; then | |
| python update_yaml.py '{"etlNginxImage":"'"${NGINX_IMAGE_NAME}:${TAG_NAME}"'","airflowImage":"'"${AIRFLOW_IMAGE_NAME}:${TAG_NAME}"'"}' apps/anyway/values-anyway-auto-updated.yaml &&\ | |
| git add apps/anyway/values-anyway-auto-updated.yaml && git commit -m "automatic update of prod etl anyway" | |
| else | |
| python update_yaml.py '{"etlNginxImageDev":"'"${NGINX_IMAGE_NAME}:${GITHUB_SHA}"'","airflowImageDev":"'"${AIRFLOW_IMAGE_NAME}:${GITHUB_SHA}"'"}' apps/anyway/values-anyway-auto-updated.yaml &&\ | |
| git add apps/anyway/values-anyway-auto-updated.yaml && git commit -m "automatic update of dev etl anyway" | |
| fi &&\ | |
| git push origin main | |
| fi | |
| fi |