refactor(airflow): airflow system load balancer using microsservices arch #667
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: EpiGraphHub | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| env: | |
| _AIRFLOW_DB_UPGRADE: true | |
| _AIRFLOW_WWW_USER_CREATE: true | |
| _AIRFLOW_WWW_USER_EMAIL: [email protected] | |
| _AIRFLOW_WWW_USER_FIRST_NAME: Super | |
| _AIRFLOW_WWW_USER_LAST_NAME: Admin | |
| _AIRFLOW_WWW_USER_PASSWORD: airflow | |
| _AIRFLOW_WWW_USER_USERNAME: airflow | |
| AIRFLOW_FILES_PATH_DIR_HOST: /home/runner/work/EpiGraphHub/EpiGraphHub/containers/airflow | |
| AIRFLOW_HOME: /opt/airflow | |
| AIRFLOW_PORT: 8099 | |
| AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth | |
| AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: true | |
| AIRFLOW__CORE__FERNET_KEY: rhZaG4rip4DrA4Z93rI9uqWVK2/8Tk+zSixmEe3ZZo8= | |
| AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://dev_airflow_user:airflow_password@postgres:25432/dev_airflow | |
| AIRFLOW__WEBSERVER__SECRET_KEY: 79197e3bef40d1d2a6cf07cb1f4c9f54e82a3bcfc6b92ed06d081856eee2 | |
| AWS_ACCESS_KEY_ID: minio-user | |
| AWS_SECRET_ACCESS_KEY: minio-password | |
| AWS_DEFAULT_REGION: us-east-1 | |
| ENV: dev | |
| FLOWER_PORT: 28888 | |
| FLOWER_PASSWORD: flowerpass | |
| MINIO_ROOT_USER: minio-user | |
| MINIO_ROOT_PASSWORD: minio-password | |
| MINIO_PORT_1: 9000 | |
| MINIO_PORT_2: 9090 | |
| POSTGRES_HOST_UID: 1001 | |
| POSTGRES_HOST_GID: 121 | |
| POSTGRES_HOST: postgres | |
| POSTGRES_PORT: 25432 | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_EPIGRAPH_HOST: postgres | |
| POSTGRES_EPIGRAPH_PORT: 25432 | |
| POSTGRES_EPIGRAPH_USER: dev_epigraph | |
| POSTGRES_EPIGRAPH_PASSWORD: dev_epigraph | |
| POSTGRES_EPIGRAPH_DB: dev_epigraphhub | |
| POSTGRES_EPIGRAPH_DB_PRIVATE: dev_privatehub | |
| POSTGRES_EPIGRAPH_DB_SANDBOX: dev_sandbox | |
| REDIS_HOST: redis | |
| REDIS_PORT: 6379 | |
| REDIS_PASSWORD: redispass | |
| SUPERSET_PORT: 8088 | |
| SUPERSET_DB_PATH_DIR_HOST: /tmp/superset | |
| SUPERSET_DB_PATH_HOST: /tmp/superset/superset.db | |
| SUPERSET_HOME: /opt/data/superset | |
| SUPERSET_MAIL_DEFAULT_SENDER: | |
| SUPERSET_MAIL_PASSWORD: | |
| SUPERSET_MAIL_PORT: 587 | |
| SUPERSET_MAIL_SERVER: | |
| SUPERSET_MAIL_USERNAME: | |
| SUPERSET_MAIL_USE_TLS: true | |
| SUPERSET_SECRET_KEY: superset-on-ci | |
| POSTGRES_DATA_DIR_HOST: /home/runner/work/psql_data | |
| POSTGRES_CONFIG_FILE_HOST: /home/runner/work/EpiGraphHub/EpiGraphHub/containers/postgresql/sql/dev/postgresql.conf | |
| SUPERSET_DB_USER: postgres | |
| SUPERSET_DB_PASSWORD: postgres | |
| SUPERSET_DB_DATABASE: postgres | |
| jobs: | |
| main: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 55 | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Create environment variables file | |
| run: | | |
| export HOST_UID=$(id -u) | |
| export HOST_GID=$(id -g) | |
| export POSTGRES_HOST_UID=$(id -u) | |
| export POSTGRES_HOST_GID=$(id -g) | |
| envsubst < .env.tpl > .env | |
| mkdir /home/runner/work/psql_data | |
| - name: setup | |
| run: | | |
| make prepare-host | |
| - uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| miniconda-version: "latest" | |
| mamba-version: "*" | |
| environment-file: conda/base.yaml | |
| channels: conda-forge,r,nodefaults | |
| activate-environment: epigraphhub | |
| use-mamba: true | |
| miniforge-variant: Mambaforge | |
| - name: configure epigraphhub credentials | |
| run: | | |
| epigraphhub-config --name ci-epigraphhub \ | |
| --db-uri "${POSTGRES_EPIGRAPH_USER}:${POSTGRES_EPIGRAPH_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_EPIGRAPH_DB}" | |
| - name: configure aws s3 credentials to minio | |
| run: make dev-create-s3-credential | |
| - name: build docker containers | |
| run: make containers-build | |
| - name: start services | |
| run: | | |
| make containers-start-services SERVICES="superset airflow postgres" | |
| - name: wait for the services are properly working | |
| run: | | |
| docker ps | |
| make containers-wait-all | |
| - name: run epigraphhub tests | |
| run: | | |
| wget https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/116.0.5845.96/linux64/chromedriver-linux64.zip -O /tmp/chromedriver-linux64.zip | |
| unzip /tmp/chromedriver-linux64.zip -d /tmp | |
| sudo rm /usr/bin/chromedriver | |
| sudo mv /tmp/chromedriver-linux64/chromedriver /usr/bin | |
| python -m unittest /home/runner/work/EpiGraphHub/EpiGraphHub/tests/* | |
| # TODO: Fix containers/postgres/sql/dev/02-epigraphhub.sql script | |
| # - name: Trigger all DAGs | |
| # run: | | |
| # make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test owid 2022-10-25' | |
| # make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test foph 2022-10-25' | |
| # make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test colombia 2022-10-25' | |
| # make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test web_status_test 2022-10-25' | |
| - name: Test MinIO access | |
| run: aws --endpoint-url http://localhost:9000/ s3 ls | |
| - name: Generate logs | |
| if: ${{ failure() }} | |
| run: | | |
| make containers-logs ARGS="--tail 1000" SERVICES="" > /tmp/containers-services.log | |
| - name: Archive log artifacts | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ failure() }} | |
| with: | |
| name: containers-services | |
| path: /tmp/containers-services.log |