build-and-test-container.yml

name: build-container
on:
  push:
    branches:
      - master
      - dev
  pull_request:
    branches:
      - master
      - dev

jobs:
  changed-files:
    runs-on: ubuntu-latest
    outputs:
      changed: ${{ steps.paths-docker-changed.outputs.any_changed }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          submodules: 'recursive'
          fetch-depth: 0
      - name: Detect if paths-ignore changed
        id: paths-docker-changed
        uses: tj-actions/changed-files@v37
        with:
          files: |
            Dockerfile
            requirements.txt
            yolov5_tracker/requirements.txt
            yolov5/requirements.txt
            
  # Based on if the files have changed and whether we push or PR, 
  # we decide on the tag of the image.
  check-image:
    runs-on: ubuntu-latest
    needs: [changed-files]
    outputs:
      tag: ${{ steps.find-image-tag.outputs.tag }}
    steps:
      - name: find-image-tag
        id: find-image-tag
        run: |
          if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ needs.changed-files.outputs.changed }}" = "false" ] ; then
            echo "tag=${{ github.base_ref }}" >> "$GITHUB_OUTPUT"
          elif [ "${{ github.event_name }}" = "pull_request" ] ; then
            echo "tag=${{ github.head_ref }}" >> "$GITHUB_OUTPUT"
          else
            echo "tag=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
          fi
        shell: bash

  check-strategy:
    # This is for the case where we want to update the code in the master container, while nothing else to the container has changed. 
    # NOTE THAT THIS IS A HACK! The way this gets updated now, is by adding an extra layer to the container with the new code. The old layer of code still exists and this will cause the image size to increase. However, this is an easy solution for now and it is unknown how much the image size increases. So for now we will leave it like this. More info on how to avoid this Hack is in issue #244.
    runs-on: ubuntu-latest
    needs: [check-image, changed-files]
    outputs:
      strategy: ${{ steps.build-strategy.outputs.strategy }}
    steps:
      - name: Determine Build Strategy
        id: build-strategy
        run: |
          if [ "${{ needs.changed-files.outputs.changed }}" = "true" ] ; then
            echo "strategy=rebuild" >> "$GITHUB_OUTPUT"
          else
            echo "strategy=update" >> "$GITHUB_OUTPUT"
          fi
        shell: bash
               
       
  build-and-push-image:
    runs-on: ubuntu-latest
    needs: [changed-files, check-image, check-strategy]
    # We always want to update image master on a push, so that the code in the container on SNIC is up to date
    if: ${{ needs.changed-files.outputs.changed == 'true' || github.ref_name == 'master' }}
    permissions:
      contents: read
      packages: write
    steps:
      - name: Free disk space (specific to Ubuntu images)
        run: |
          sudo rm -rf /usr/share/dotnet || true
          sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          submodules: 'recursive'
      - name: Log in to the Container registry
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: '${{ github.actor }}'
          password: '${{ secrets.GITHUB_TOKEN }}'
      
      - name: Build Docker Image
        # It will only go into the else statement, when we build a new container for master when nothing to the docker environment has changed.
        # In that case we only want to update the code inside of the container. NOTE THAT THIS IS A HACK, see explanation above.
        run: |
          if [[ "${{ needs.check-strategy.outputs.strategy }}" == "rebuild" ]] ; then
            docker build \
              -t ghcr.io/${{ github.repository }}:${{ needs.check-image.outputs.tag }} \
              -t ghcr.io/${{ github.repository }}:latest .
          else
            echo "FROM ghcr.io/${{ github.repository }}:${{ needs.check-image.outputs.tag }}" > Dockerfile.update
            echo "COPY . ./kso" >> Dockerfile.update
            docker pull ghcr.io/${{ github.repository }}:latest
            docker build \
              -t ghcr.io/${{ github.repository }}:${{ needs.check-image.outputs.tag }} \
              -t ghcr.io/${{ github.repository }}:latest \
              -f Dockerfile.update .
          fi
          
        # For future, after docker file is optimized more, in the first if statement, add:
        # docker pull ghcr.io/${{ github.repository }}:${{ needs.check-image.outputs.tag }}
        # This is to reuse layers. 


      - name: Push Docker Image
        run: docker push --all-tags ghcr.io/${{ github.repository }}


  # Get the gitlab runner id to be able to checkout the repository.
  # Otherwise you do not have permission.
  configure:
    runs-on: ubuntu-latest
    outputs:
      uid_gid: ${{ steps.get-user.outputs.uid_gid }}
    steps:
      - id: get-user
        run: echo "uid_gid=$(id -u):$(id -g)" >> $GITHUB_OUTPUT

  tests:
    runs-on: ubuntu-latest
    if: ${{ always() }}
    needs: [build-and-push-image, configure, check-image]
    container:
      image: ghcr.io/${{ github.repository }}:${{ needs.check-image.outputs.tag }}
      options: --user ${{ needs.configure.outputs.uid_gid }}
      credentials:
        username: '${{ github.actor }}'
        password: '${{ secrets.GITHUB_TOKEN }}'
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Initialize conda and activate environment
        shell: bash  # Ensure the shell is bash
        id: activate-conda
        run: |
          source /opt/conda/etc/profile.d/conda.sh
          conda activate myenv
          echo "PATH=$(conda env list | grep 'myenv' | awk '{print $NF}')/bin:$PATH" >> $GITHUB_ENV
          echo "PYTHONPATH=$(conda env list | grep 'myenv' | awk '{print $NF}')/lib/python3.8/site-packages:$PYTHONPATH" >> $GITHUB_ENV

      - name: Install pylint, pytest, and nbqa
        shell: bash
        run: |
          pip install pylint pytest nbqa
          echo "/github/home/.local/bin" >> $GITHUB_PATH  # Add the local bin path to the GitHub path

      - name: Pylint Test kso_utils
        shell: bash
        run: |
          pylint --disable=possibly-used-before-assignment,used-before-assignment,no-member,assignment-from-no-return kso_utils
      - name: Pylint Test tests
        if: success() || failure()
        shell: bash
        run: |
          pylint --disable=no-member,assignment-from-no-return test/*
      - name: Pylint Test NB notebooks
        if: success() || failure()
        shell: bash
        run: |
          nbqa pylint --disable=unexpected-keyword-arg,no-member,assignment-from-no-return,no-value-for-parameter notebooks
        # If one of the pylint steps fails, there are errors and we do not need to run the test below.
      - name: Login W&B
        shell: bash
        env:
          WANDB_API: ${{ secrets.WANDB_KEY }}
        run: |
          source /opt/conda/etc/profile.d/conda.sh
          conda activate myenv 
          wandb login "$WANDB_API"
      - name: Run notebook and widget tests
        id: nb-tests
        run: >-
          export "WANDB_DIR=$(mktemp -d)" && export WANDB_CACHE_DIR="$WANDB_DIR" &&
          mkdir -p yolov5_tracker/trackers && cp src/multi_tracker_zoo.py yolov5_tracker/trackers/multi_tracker_zoo.py &&
          pytest --disable-warnings test/widget-tests.py &&
          pytest --disable-warnings --zoo_user=${{ secrets.ZOO_USERNAME }} --zoo_pass=${{ secrets.ZOO_TOKEN }} test/notebook-tests.py