main.yaml

name: EpiGraphHub

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

env:
  _AIRFLOW_DB_UPGRADE: true
  _AIRFLOW_WWW_USER_CREATE: true
  _AIRFLOW_WWW_USER_EMAIL: admin@epigraphhub.org
  _AIRFLOW_WWW_USER_FIRST_NAME: Super
  _AIRFLOW_WWW_USER_LAST_NAME: Admin
  _AIRFLOW_WWW_USER_PASSWORD: airflow
  _AIRFLOW_WWW_USER_USERNAME: airflow
  AIRFLOW_FILES_PATH_DIR_HOST: /home/runner/work/EpiGraphHub/EpiGraphHub/containers/airflow
  AIRFLOW_HOME: /opt/airflow
  AIRFLOW_PORT: 8099
  AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth
  AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: true
  AIRFLOW__CORE__FERNET_KEY: rhZaG4rip4DrA4Z93rI9uqWVK2/8Tk+zSixmEe3ZZo8=
  AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://dev_airflow_user:airflow_password@postgres:25432/dev_airflow
  AIRFLOW__WEBSERVER__SECRET_KEY: 79197e3bef40d1d2a6cf07cb1f4c9f54e82a3bcfc6b92ed06d081856eee2
  AWS_ACCESS_KEY_ID: minio-user
  AWS_SECRET_ACCESS_KEY: minio-password
  AWS_DEFAULT_REGION: us-east-1
  ENV: dev
  FLOWER_PORT: 28888
  FLOWER_PASSWORD: flowerpass
  MINIO_ROOT_USER: minio-user
  MINIO_ROOT_PASSWORD: minio-password
  MINIO_PORT_1: 9000
  MINIO_PORT_2: 9090
  POSTGRES_HOST_UID: 1001
  POSTGRES_HOST_GID: 121
  POSTGRES_HOST: postgres
  POSTGRES_PORT: 25432
  POSTGRES_USER: postgres
  POSTGRES_PASSWORD: postgres
  POSTGRES_DB: postgres
  POSTGRES_EPIGRAPH_HOST: postgres
  POSTGRES_EPIGRAPH_PORT: 25432
  POSTGRES_EPIGRAPH_USER: dev_epigraph
  POSTGRES_EPIGRAPH_PASSWORD: dev_epigraph
  POSTGRES_EPIGRAPH_DB: dev_epigraphhub
  POSTGRES_EPIGRAPH_DB_PRIVATE: dev_privatehub
  POSTGRES_EPIGRAPH_DB_SANDBOX: dev_sandbox
  REDIS_HOST: redis
  REDIS_PORT: 6379
  REDIS_PASSWORD: redispass
  SUPERSET_PORT: 8088
  SUPERSET_DB_PATH_DIR_HOST: /tmp/superset
  SUPERSET_DB_PATH_HOST: /tmp/superset/superset.db
  SUPERSET_HOME: /opt/data/superset
  SUPERSET_MAIL_DEFAULT_SENDER:
  SUPERSET_MAIL_PASSWORD:
  SUPERSET_MAIL_PORT: 587
  SUPERSET_MAIL_SERVER:
  SUPERSET_MAIL_USERNAME:
  SUPERSET_MAIL_USE_TLS: true
  SUPERSET_SECRET_KEY: superset-on-ci
  POSTGRES_DATA_DIR_HOST: /home/runner/work/psql_data
  POSTGRES_CONFIG_FILE_HOST: /home/runner/work/EpiGraphHub/EpiGraphHub/containers/postgresql/sql/dev/postgresql.conf
  SUPERSET_DB_USER: postgres
  SUPERSET_DB_PASSWORD: postgres
  SUPERSET_DB_DATABASE: postgres

jobs:
  main:

    runs-on: ubuntu-latest
    timeout-minutes: 55
    defaults:
      run:
        shell: bash -l {0}
    concurrency:
      group: ci-${{ github.ref }}
      cancel-in-progress: true

    steps:
    - uses: actions/checkout@v2

    - name: Create environment variables file
      run: |
        export HOST_UID=$(id -u)
        export HOST_GID=$(id -g)
        export POSTGRES_HOST_UID=$(id -u)
        export POSTGRES_HOST_GID=$(id -g)
        envsubst < .env.tpl > .env
        mkdir /home/runner/work/psql_data

    - name: setup
      run: |
        make prepare-host

    - uses: conda-incubator/setup-miniconda@v2
      with:
        miniconda-version: "latest"
        mamba-version: "*"
        environment-file: conda/base.yaml
        channels: conda-forge,r,nodefaults
        activate-environment: epigraphhub
        use-mamba: true
        miniforge-variant: Mambaforge

    - name: configure epigraphhub credentials
      run: |
        epigraphhub-config --name ci-epigraphhub \
          --db-uri "${POSTGRES_EPIGRAPH_USER}:${POSTGRES_EPIGRAPH_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_EPIGRAPH_DB}"

    - name: configure aws s3 credentials to minio
      run: make dev-create-s3-credential

    - name: build docker containers
      run: make containers-build

    - name: start services
      run: |
        make containers-start-services SERVICES="superset airflow postgres"      

    - name: wait for the services are properly working
      run: |
        docker ps
        make containers-wait-all

    - name: run epigraphhub tests
      run: |
        wget https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/116.0.5845.96/linux64/chromedriver-linux64.zip -O /tmp/chromedriver-linux64.zip
        unzip /tmp/chromedriver-linux64.zip -d /tmp
        sudo rm /usr/bin/chromedriver
        sudo mv /tmp/chromedriver-linux64/chromedriver /usr/bin
        python -m unittest /home/runner/work/EpiGraphHub/EpiGraphHub/tests/*

    # TODO: Fix containers/postgres/sql/dev/02-epigraphhub.sql script
    # - name: Trigger all DAGs
    #   run: |
    #     make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test owid 2022-10-25'
    #     make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test foph 2022-10-25'
    #     make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test colombia 2022-10-25'
    #     make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test web_status_test 2022-10-25'

    - name: Test MinIO access
      run: aws --endpoint-url http://localhost:9000/ s3 ls

    - name: Generate logs
      if: ${{ failure() }}
      run: |
        make containers-logs ARGS="--tail 1000" SERVICES="" > /tmp/containers-services.log

    - name: Archive log artifacts
      uses: actions/upload-artifact@v3
      if: ${{ failure() }}
      with:
        name: containers-services
        path: /tmp/containers-services.log