attempt to have logging, download mapping files from s3 #1571
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: WDL CI | |
| on: | |
| push | |
| env: | |
| LC_ALL: C.UTF-8 | |
| LANG: C.UTF-8 | |
| DEBIAN_FRONTEND: noninteractive | |
| jobs: | |
| linters: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/cache@v1 | |
| with: | |
| path: ~/.cache | |
| key: ${{runner.os}}-cache | |
| - name: dependencies | |
| run: | | |
| source /etc/profile | |
| sudo apt-get -qq update | |
| sudo apt-get -qq install --yes jq moreutils make virtualenv zip unzip httpie git shellcheck | |
| virtualenv --python=python3.10 .venv | |
| source .venv/bin/activate | |
| pip install -r requirements-dev.txt | |
| - name: linters | |
| run: | | |
| source .venv/bin/activate | |
| make lint | |
| - uses: andymckay/[email protected] | |
| if: ${{ failure() }} | |
| cancel-previous: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: styfle/[email protected] | |
| with: | |
| access_token: ${{ github.token }} | |
| list-workflow-dirs: | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| matrix: ${{steps.list_dirs.outputs.matrix}} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 2 | |
| - id: list_dirs | |
| run: echo "matrix=$(./scripts/diff-workflows.sh |jq -cnR '[inputs|select(length>0)]')" >> $GITHUB_OUTPUT | |
| wdl-ci: | |
| runs-on: ubuntu-22.04 | |
| needs: list-workflow-dirs | |
| if: ${{ needs['list-workflow-dirs'].outputs.matrix != '[]' && needs['list-workflow-dirs'].outputs.matrix != '' }} | |
| strategy: | |
| matrix: | |
| workflow_dir: ${{fromJson(needs['list-workflow-dirs'].outputs.matrix)}} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: docker login ghcr.io | |
| uses: docker/login-action@v1 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: docker build + push to ghcr.io | |
| run: | | |
| TAG=$(git describe --long --tags --always --dirty) | |
| IMAGE_NAME=czid-${{ matrix.workflow_dir }}-public | |
| IMAGE_URI="ghcr.io/${GITHUB_REPOSITORY}/${IMAGE_NAME}" | |
| CACHE_FROM=""; docker pull "$IMAGE_URI" && CACHE_FROM="--cache-from $IMAGE_URI" | |
| ./scripts/docker-build.sh "workflows/${{ matrix.workflow_dir }}" --tag "${IMAGE_URI}:${TAG}" $CACHE_FROM \ | |
| || ./scripts/docker-build.sh "workflows/${{ matrix.workflow_dir }}" --tag "${IMAGE_URI}:${TAG}" | |
| docker push "${IMAGE_URI}:${TAG}" | |
| if [[ ${GITHUB_REF##*/} == "main" ]]; then | |
| docker tag "${IMAGE_URI}:${TAG}" "${IMAGE_URI}:latest" | |
| docker push "${IMAGE_URI}:latest" | |
| fi | |
| echo "IMAGE_URI=${IMAGE_URI}" >> $GITHUB_ENV | |
| echo "TAG=${TAG}" >> $GITHUB_ENV | |
| - name: run tests | |
| run: | | |
| # explicitly block EC2 IMDS endpoint to work around awscli issue: | |
| # https://github.com/aws/aws-cli/issues/5234#issuecomment-635459464 | |
| # https://github.com/aws/aws-cli/issues/5262 | |
| sudo ip route add blackhole 169.254.169.254 | |
| # configure miniwdl to auto-delete task working directories, to reduce chance of worker | |
| # running out of space | |
| export MINIWDL__FILE_IO__OUTPUT_HARDLINKS=true | |
| export MINIWDL__FILE_IO__DELETE_WORK=success | |
| export MINIWDL__DOWNLOAD_CACHE__GET=true | |
| export MINIWDL__DOWNLOAD_CACHE__PUT=true | |
| export MINIWDL__DOWNLOAD_CACHE__DIR="$(mktemp -d --tmpdir miniwdl_download_cache_XXXXXX)" | |
| source /etc/profile | |
| sudo apt-get -qq update | |
| sudo apt-get -qq install --yes jq moreutils make virtualenv zip unzip httpie git shellcheck | |
| virtualenv --python=python3.10 .venv | |
| source .venv/bin/activate | |
| pip install -r requirements-dev.txt | |
| export DOCKER_IMAGE_ID="${IMAGE_URI}:${TAG}" | |
| make test-${{ matrix.workflow_dir }} |