python-ci.yml

name: Python CI
on:
  pull_request:
    paths:
      - python/**

  push:
    branches:
      - main
    paths:
      - python/**

  workflow_dispatch:

env:
  LOOKERSDK_BASE_URL: https://localhost:20000
  LOOKERSDK_VERIFY_SSL: false
  TOX_JUNIT_OUTPUT_DIR: results

jobs:
  typecheck:
    name: Mypy
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
        working-directory: python/

    steps:
      - name: Cancel Previous Runs
        uses: styfle/cancel-workflow-action@0.6.0
        with:
          access_token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions/checkout@v2
      - uses: actions/setup-python@v2
        with:
          python-version: 3.9.9
      - run: pip install -e .
      - run: pip install mypy types-requests
      - run: mypy looker_sdk/

  unit:
    needs: typecheck
    name: Unit - ${{ matrix.os }} / py${{ matrix.python-version }}
    env:
      TOX_JUNIT_OUTPUT_NAME: ${{ matrix.os }}.py${{ matrix.python-version }}
    runs-on: ${{ matrix.os }}-latest
    defaults:
      run:
        shell: bash
        working-directory: python/

    strategy:
      # run all supported python versions on ubuntu
      # run only latest supported python version on windows/mac
      matrix:
        os:
          - ubuntu
          - macos
          - windows
        python-version:
          - '3.9.9'
        include:
          - python-version: '3.6'
            os: ubuntu
          - python-version: '3.7'
            os: ubuntu
          - python-version: '3.8'
            os: ubuntu

    steps:
      - name: Repo Checkout
        uses: actions/checkout@v2

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Tox and any other packages
        run: |
          python -m pip install --upgrade pip
          pip install tox

      - name: Run Unit Tests
        run: tox -e unit

      - name: Upload pytest test results
        if: ${{ always() }}
        uses: actions/upload-artifact@v2
        with:
          name: python-test-results
          path: python/results/

  integration:
    needs: unit
    name: Integration - ${{ matrix.os }} / Looker.${{ matrix.looker }}
    env:
      TOX_JUNIT_OUTPUT_NAME: ${{ matrix.os }}.Looker-${{ matrix.looker }}.py3.x
    runs-on: ${{ matrix.os }}-latest
    defaults:
      run:
        shell: bash
        working-directory: python/

    strategy:
      matrix:
        os:
          - ubuntu
        looker:
          - '21_18'
          - '21_20'
          - '22_0'
          - '22_2'
          # TODO uncomment `include:` when either macos or windows works to satisfaction.
          #include:
          # TODO: macos matrix leg is functional but it takes ~20 minutes (compared
          # to ~4 minutes for ubuntu) because docker install takes ~5 minutes
          # and docker pull takes ~10 minutes. We can probably figure out how to
          # cache the docker install but hopefully github will soon have docker
          # available pre-installed on macos so not worth the effort now.
          # Regarding docker pull ... it would be nice if there's a way to cache
          # only some layers of the image on the runner but we don't want to cache
          # the final layer(s) with Looker IP. This would speed up docker pull on
          # all OSs.
          #- os: macos
          #  python-version: 3.9
          #  looker: '7_20'
          # TODO: currently can't run linux containers on windows.
          # Pending new windows server version
          # https://github.com/actions/virtual-environments/issues/1143#issuecomment-698797524
          #- os: windows
          #  python-version: 3.9
          #  looker: '7_20'
    steps:
      - name: Repo Checkout
        uses: actions/checkout@v2

      - name: Set up Cloud SDK
        uses: google-github-actions/setup-gcloud@v0.2.0
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
          service_account_key: ${{ secrets.GCP_AR_READER_SA_KEY }}
          export_default_credentials: true

      - name: Authenticate Artifact Repository
        run: gcloud auth configure-docker us-west1-docker.pkg.dev --quiet

      - name: Install docker on macos
        if: ${{ matrix.os == 'macos' }}
        uses: docker-practice/actions-setup-docker@v1
        with:
          docker_channel: stable
          docker_buildx: false
          docker_cli_experimental: disabled

      - name: Bump docker for mac memory
        if: ${{ matrix.os == 'macos' }}
        run: |
          osascript -e 'quit app "Docker"'
          sed -i'.original' -e's/  "memoryMiB" : 2048/  "memoryMiB" : 8192/' ~/Library/Group\ Containers/group.com.docker/settings.json
          open -g /Applications/Docker.app
          # re-run docker startup logic from docker-practice/actions-setup-docker action
          sleep 60
          i=0
          while ! docker system info &>/dev/null; do
          (( i++ == 0 )) && printf %s '-- Waiting for Docker to finish starting up...' || printf '.'
          sleep 1
          done
          (( i )) && printf '\n'
          echo "-- Docker is ready."

      - name: Pull and run Looker docker image
        # TODO: can we cache some layers of the image for faster download?
        # we probably don't want to cache the final image for IP security...
        run: |
          docker pull --quiet us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/${{ matrix.looker }}
          # set $LOOKER_OPTS to --no-ssl if we want to turn off ssl
          docker run --name looker-sdk-codegen-ci -d -p 10000:9999 -p 20000:19999 us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/${{ matrix.looker }}
          docker logs -f looker-sdk-codegen-ci --until=30s &

      - name: Install Tox and any other packages
        run: |
          python -m pip install --upgrade pip
          pip install tox

      - name: Determine credentials version
        # Prior to 21_18, each version had different credentials and a 
        # different secret. 21_20 and later all use the same credentials
        # as 21_18. The parse_version.sh script parses the version and
        # yields 21_12, 21_14, 21_16 etc for those versions but 21_18 for
        # anything 21_18, 21_20, 22_0, etc.
        #
        # This can be factored out entirely once 21_18 is the earliest
        # supported looker version.
        run: |
          echo "CREDENTIALS_VERSION=$(${{ github.workspace }}/.github/scripts/parse_version.sh <<< ${{ matrix.looker }})" >> $GITHUB_ENV

      - name: Set up Looker credentials
        # Load the correct credentials based on the version from
        # the prior step.
        run: |
          echo "LOOKERSDK_CLIENT_ID=${{ secrets[format('LOOKERSDK_CLIENT_ID__{0}', env.CREDENTIALS_VERSION )] }}" >> $GITHUB_ENV
          echo "LOOKERSDK_CLIENT_SECRET=${{ secrets[format('LOOKERSDK_CLIENT_SECRET__{0}', env.CREDENTIALS_VERSION )] }}" >> $GITHUB_ENV

      - name: Check that Looker is ready
        run: |
          ${{ github.workspace }}/.github/scripts/wait_for_looker.sh

      - name: Run Integration Tests
        run: tox -e integration

      - name: Upload pytest test results
        if: ${{ always() }}
        uses: actions/upload-artifact@v2
        with:
          name: python-test-results
          path: python/results/

  twine:
    needs: integration
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
        working-directory: python/

    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-python@v2
        with:
          python-version: 3.9.9
      - name: Twine upload check
        run: |
          pip install wheel twine
          python setup.py sdist bdist_wheel
          twine check dist/*

  publish-test-results:
    needs: [unit, integration, twine]
    if: success() || failure()
    runs-on: ubuntu-latest

    steps:
      - name: Download Artifacts
        uses: actions/download-artifact@v2
        with:
          path: artifacts

      # If there are no test result files the test is reported as passing!
      # That allows for some weird failure modes, where an early failure
      # is not distinguished from a pass.
      - name: Check Results Files Exist
        id: check-results-exist
        run: |
          if [[ $(ls -l artifacts/python-test-results/*.xml 2> /dev/null | wc -l) -ge "1" ]]; then
            exit 0
          else
            curl --request POST \
            --url https://api.github.com/repos/looker-open-source/sdk-codegen/check-runs \
            --header "authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
            --header "content-type: application/json" \
            --header "Accept: application/vnd.github.v3+json" \
            --data '{
              "name": "Python Tests",
              "head_sha": "${{ github.event.pull_request.head.sha || github.sha }}",
              "conclusion": "failure"
            }' \
            --fail \
            -o /dev/null \
            --silent
            exit 1
          fi

      - name: Publish Unit Test Results
        if: steps.check-results-exist.outcome == 'success'
        uses: EnricoMi/publish-unit-test-result-action@v1.12
        with:
          # Cosmetic issue with `check_name` being associated to the wrong
          # workflow: https://github.com/EnricoMi/publish-unit-test-result-action/issues/12
          check_name: Python Tests
          github_token: ${{ secrets.GITHUB_TOKEN }}
          report_individual_runs: true
          hide_comments: orphaned commits
          check_run_annotations_branch: '*'
          files: 'artifacts/python-test-results/*.xml'