docker.yml

name: Internet.nl Docker

on:
  pull_request:
  release:
    types: [published]
  push:
    branches:
      - main
      - release/*

env:
  DOCKER_REGISTRY: ghcr.io/${{ github.repository_owner }}
  # determine whether this pull request has permissions to push to registry, or artifacts
  # should be used to transfer images between jobs. Forked and dependabot builds don't
  # have permission to push to registry.
  use_registry: ${{ ! (github.event_name == 'pull_request' && (github.event.pull_request.head.repo.full_name != github.repository || startsWith(github.head_ref, 'dependabot/'))) }}

jobs:
  # builds all docker images in parallel
  build-docker:
    runs-on: ubuntu-22.04

    strategy:
      matrix:
        include:
          - image: internet.nl
            dockerfile: docker/Dockerfile
            target: app
          - image: unbound
            dockerfile: docker/Dockerfile
            target: unbound
          - image: linttest
            dockerfile: docker/Dockerfile
            target: linttest
          - image: test-runner
            dockerfile: docker/test-runner.Dockerfile
            target:
          - image: webserver
            dockerfile: docker/webserver.Dockerfile
            target:
          - image: grafana
            dockerfile: docker/grafana.Dockerfile
            target:
          - image: util
            dockerfile: docker/util.Dockerfile
            target:

    outputs:
      internetnl_version: ${{ steps.get_version.outputs.internetnl_version }}

    steps:
      - name: Debug info
        run: |
          docker version --format '{{.Server.Version}}'
          docker compose version --short

      - uses: actions/checkout@v4
        # include vendor/ submodules used to build dependencies like nassl and unbound
        with:
          submodules: recursive

      - name: Unshallow repository for version tag
        run: |
          # https://github.com/pypa/setuptools_scm/issues/414
          git fetch --prune --unshallow
          # don't fail on this command if there are no tags (eg, on forked repos)
          git fetch --depth=1 origin +refs/tags/*:refs/tags/* || true

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Generate version number
        id: get_version
        run: |
          pip -q install setuptools_scm
          # '+' is not supported in Docker Image tags
          scm_version=$(echo 'from setuptools_scm import get_version; print(get_version(version_scheme="release-branch-semver"))' | python)
          branch_version_suffix=${GITHUB_HEAD_REF:+-$GITHUB_HEAD_REF}
          echo "internetnl_version=$scm_version$branch_version_suffix" | tr '+' '-'| tee -a "$GITHUB_OUTPUT"

      # login to pull images from Github registry
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Login to Docker Hub to increase rate limit on pulls
        # https://github.com/docker/login-action?tab=readme-ov-file#dockerhub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
        # worst case of this failing is running into Docker Hub rate limit
        continue-on-error: true

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build ${{ matrix.image }} (for non-forked PR's)
        # build for non-forked PR's that are allowed to use the registry
        if: ${{ env.use_registry == 'true' }}
        # build steps should not take longer than 15 minutes, if they do it's probably because Github Actions hangs
        timeout-minutes: 15
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ${{ matrix.dockerfile }}
          target: ${{ matrix.target }}
          # tag image with current setuptools_scm generated version
          # and tag with PR source branch (eg: feature-x)
          tags: |
            ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:${{ steps.get_version.outputs.internetnl_version }}
            ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:pr-${{ github.event_name == 'pull_request' && github.event.number || github.ref_name}}
          # use latest build from main, or image previously build by this PR for caching
          cache-from: |
            ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:main
            ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:pr-${{ github.event_name == 'pull_request' && github.event.number || github.ref_name}}
          # push images to registry
          push: true
          # makes build images better usable as cache by allowing individual layers to be pulled from cache
          # pass in version information
          build-args: |
            BUILDKIT_INLINE_CACHE=1
            RELEASE=${{ steps.get_version.outputs.internetnl_version }}

      - name: Build ${{ matrix.image }} (for main)
        # build for pushes to the main branch
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        # build steps should not take longer than 15 minutes, if they do it's probably because Github Actions hangs
        timeout-minutes: 15
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ${{ matrix.dockerfile }}
          target: ${{ matrix.target }}
          # tag image with current setuptools_scm generated version
          # and tag with current branch name (eg: main)
          tags: |
            ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:${{ steps.get_version.outputs.internetnl_version }}
            ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:main
          # use latest build from main for caching
          cache-from: |
            ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:main
          # push images to registry
          push: true
          # makes build images better usable as cache by allowing individual layers to be pulled from cache
          # pass in version information
          build-args: |
            BUILDKIT_INLINE_CACHE=1
            RELEASE=${{ steps.get_version.outputs.internetnl_version }}

      - name: Build ${{ matrix.image }} (for release)
        # build for tagged releases
        if: github.event_name == 'release'
        # build steps should not take longer than 15 minutes, if they do it's probably because Github Actions hangs
        timeout-minutes: 15
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ${{ matrix.dockerfile }}
          target: ${{ matrix.target }}
          # tag image with current setuptools_scm generated version and tag 'latest'
          tags: |
            ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:${{ steps.get_version.outputs.internetnl_version }}
            ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:latest
          # use latest build from main for caching
          cache-from: |
            ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:main
          # push images to registry
          push: true
          # makes build images better usable as cache by allowing individual layers to be pulled from cache
          # pass in version information
          build-args: |
            BUILDKIT_INLINE_CACHE=1
            RELEASE=${{ steps.get_version.outputs.internetnl_version }}

      - name: Build ${{ matrix.image }} (for forked PR's or dependabot)
        # build for forked PR's that don't have permissions to push to the container registry or dependabot PR's
        if: ${{ env.use_registry == 'false' }}
        # build steps should not take longer than 15 minutes, if they do it's probably because Github Actions hangs
        timeout-minutes: 15
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ${{ matrix.dockerfile }}
          target: ${{ matrix.target }}
          # tag image with current setuptools_scm generated version
          tags: |
            ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:${{ steps.get_version.outputs.internetnl_version }}
          # use latest build from main for caching
          cache-from: |
            ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:main
          # don't push to registry due to permissions, but store in Docker so can be exported for artifacts below
          load: true
          # makes build images better usable as cache by allowing individual layers to be pulled from cache
          # pass in version information
          build-args: |
            RELEASE=${{ steps.get_version.outputs.internetnl_version }}

      - name: Save image to disk (for forked PR's or dependabot)
        # trigger only for forked PR's that don't have permissions to push to the container registry
        if: ${{ env.use_registry == 'false' }}
        run: docker save ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:${{ steps.get_version.outputs.internetnl_version }} | gzip > "${{ matrix.image }}.tar.gz"

      - name: Upload image as build artifact (for forked PR's or dependabot)
        # trigger only for forked PR's that don't have permissions to push to the container registry
        if: ${{ env.use_registry == 'false' }}
        uses: actions/upload-artifact@v4
        with:
          name: "${{ matrix.image }}"
          path: "${{ matrix.image }}.tar.gz"
          # we don't need to keep these any longer than the subsequent jobs, this is the shortest it can be configured
          retention-days: 1

  docs:
    runs-on: ubuntu-22.04
    needs: [build-docker]
    steps:
      - name: Branch deployment docs
        run: |
          cat >> $GITHUB_STEP_SUMMARY <<EOF

          To deploy this release to a existing deployment run the following update command:

              docker run -ti --rm --pull=always \\
                --volume /var/run/docker.sock:/var/run/docker.sock \\
                --volume \$HOME/.docker:/root/.docker \\
                --volume /opt/Internet.nl:/opt/Internet.nl \\
                --network none \\
                --env DOCKER_REGISTRY=${{ env.DOCKER_REGISTRY }} \\
                ${{ env.DOCKER_REGISTRY }}/util:${{ needs.build-docker.outputs.internetnl_version }} \\
                /deploy.sh
          EOF

  integration-test:
    needs: [build-docker]

    runs-on: ubuntu-24.04

    env:
      # used in `compose.yaml` files to determine version of images to pull
      RELEASE: "${{ needs.build-docker.outputs.internetnl_version }}"
      PY_COLORS: "1"

    steps:
      - name: Install specific Docker/Compose versions that are known to work
        run: |
          # install Docker apt repository
          sudo apt update
          sudo apt-get install ca-certificates curl
          sudo install -m 0755 -d /etc/apt/keyrings
          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
          sudo chmod a+r /etc/apt/keyrings/docker.asc
          echo \
            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
            $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
            sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
          sudo apt-get update

          # upgrade Docker
          sudo apt install --upgrade docker-ce docker-compose-plugin=2.27.0\*

      - name: Debug info
        run: |
          docker version --format '{{.Server.Version}}'
          docker compose version --short

      - name: Enable ip6tables in Docker
        run: |
          sudo bash -c 'echo "{ \"ip6tables\": true, \"experimental\":true}" > /etc/docker/daemon.json'
          sudo systemctl restart docker.service
          sudo ip6tables -I DOCKER-USER --dst ff00::/8 -j ACCEPT

      - uses: actions/checkout@v4

      # login to pull images from Github registry
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Login to Docker Hub to increase rate limit on pulls
        # https://github.com/docker/login-action?tab=readme-ov-file#dockerhub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
        # worst case of this failing is running into Docker Hub rate limit
        continue-on-error: true

      - name: Download images from artifacts (for forked PR's or dependabot)
        # trigger only for forked PR's that don't have permissions to push to the container registry
        if: ${{ env.use_registry == 'false' }}
        uses: actions/download-artifact@v4
        with:
          path: images/

      - name: Load images from artifacts (for forked PR's or dependabot)
        # trigger only for forked PR's that don't have permissions to push to the container registry
        if: ${{ env.use_registry == 'false' }}
        run: find images/ -type f -name *.tar.gz -exec  sh -c 'gunzip --stdout "{}" | docker load' \;

      - name: Pull additional docker images
        if: ${{ env.use_registry == 'false' }}
        # build env includes all images, this will pull additional images not loaded from artifacts for
        # forked PR's/dependabot build
        run: make pull env=build pull_args=--ignore-buildable

      - name: Pull all docker images
        if: ${{ env.use_registry == 'true' }}
        # build env includes all images, this will pull additional images from the registry for non-forked PR builds
        # main and release builds
        run: make pull env=build

      - name: Start test instance
        run: make up env=test

      - name: Run integration tests
        run: make integration-tests-verbose env=test

      - name: Check nginx config
        run: make check-gixy

      - name: Collect Docker Compose logs
        if: always()
        run: make logs-all-dump env=test > docker-compose.log

      - uses: test-summary/action@v2.3
        with:
          paths: test-results.xml
        if: always()
        continue-on-error: true

      - name: Failure log
        if: failure()
        # log last few lines in case of failure for quick debugging
        run: make docker-compose args="logs --tail=100" env=test

      - name: Archive test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: Playwright integration test results (screenshots, video) ${{ matrix.os }}
          path: test-results/
          if-no-files-found: ignore

      - name: Archive test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: Integration test Docker Compose Logs ${{matrix.os}}
          path: docker-compose.log
          if-no-files-found: ignore

  lintcheck:
    name: lint/check
    needs: [build-docker]
    runs-on: ubuntu-22.04

    env:
      # used in `compose.yaml` files to determine version of images to pull
      RELEASE: "${{ needs.build-docker.outputs.internetnl_version }}"

    steps:
      - name: Debug info
        run: |
          docker version --format '{{.Server.Version}}'
          docker compose version --short

      - uses: actions/checkout@v4

      # login to pull images from Github registry
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Login to Docker Hub to increase rate limit on pulls
        # https://github.com/docker/login-action?tab=readme-ov-file#dockerhub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
        # worst case of this failing is running into Docker Hub rate limit
        continue-on-error: true

      - name: Download images from artifacts (for forked PR's or dependabot)
        # trigger only for forked PR's that don't have permissions to push to the container registry
        if: ${{ env.use_registry == 'false' }}
        uses: actions/download-artifact@v4
        with:
          path: images/

      - name: Load images from artifacts (for forked PR's or dependabot)
        # trigger only for forked PR's that don't have permissions to push to the container registry
        if: ${{ env.use_registry == 'false' }}
        run: find images/ -type f -name *.tar.gz -exec  sh -c 'gunzip --stdout "{}" | docker load' \;

      - name: Run check
        run: /bin/bash -o pipefail -c 'make --silent check | tee -a $GITHUB_STEP_SUMMARY'

      - name: Run lint
        run: /bin/bash -o pipefail -c 'make --silent lint | tee -a $GITHUB_STEP_SUMMARY'

  test:
    needs: [build-docker]
    runs-on: ubuntu-24.04

    env:
      # used in `compose.yaml` files to determine version of images to pull
      RELEASE: "${{ needs.build-docker.outputs.internetnl_version }}"
      PY_COLORS: "1"

    steps:
      - name: Install specific Docker/Compose versions that are known to work
        run: |
          # install Docker apt repository
          sudo apt update
          sudo apt-get install ca-certificates curl
          sudo install -m 0755 -d /etc/apt/keyrings
          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
          sudo chmod a+r /etc/apt/keyrings/docker.asc
          echo \
            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
            $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
            sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
          sudo apt-get update

          # upgrade Docker
          sudo apt install --upgrade docker-ce docker-compose-plugin=2.27.0\*

      - name: Debug info
        run: |
          docker version --format '{{.Server.Version}}'
          docker compose version --short

      - uses: actions/checkout@v4

      # login to pull images from Github registry
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Login to Docker Hub to increase rate limit on pulls
        # https://github.com/docker/login-action?tab=readme-ov-file#dockerhub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
        # worst case of this failing is running into Docker Hub rate limit
        continue-on-error: true

      - name: Download images from artifacts (for forked PR's or dependabot)
        # trigger only for forked PR's that don't have permissions to push to the container registry
        if: ${{ env.use_registry == 'false' }}
        uses: actions/download-artifact@v4
        with:
          path: images/

      - name: Load images from artifacts (for forked PR's or dependabot)
        # trigger only for forked PR's that don't have permissions to push to the container registry
        if: ${{ env.use_registry == 'false' }}
        run: find images/ -type f -name *.tar.gz -exec  sh -c 'gunzip --stdout "{}" | docker load' \;

      - name: Pull additional docker images
        if: ${{ env.use_registry == 'false' }}
        # build env includes all images, this will pull additional images not loaded from artifacts for
        # forked PR's/dependabot build
        run: make pull env=build pull_args=--ignore-buildable

      - name: Pull all docker images
        if: ${{ env.use_registry == 'true' }}
        # build env includes all images, this will pull additional images from the registry for non-forked PR builds
        # main and release builds
        run: make pull env=build

      - name: Run test
        run: make test

      - uses: test-summary/action@v2.3
        with:
          paths: test-results.xml
        if: always()
        continue-on-error: true

  development-environment-test:
    needs: [build-docker]
    runs-on: ubuntu-24.04

    env:
      # used in `compose.yaml` files to determine version of images to pull
      RELEASE: "${{ needs.build-docker.outputs.internetnl_version }}"

    steps:
      - name: Install specific Docker/Compose versions that are known to work
        run: |
          # install Docker apt repository
          sudo apt update
          sudo apt-get install ca-certificates curl
          sudo install -m 0755 -d /etc/apt/keyrings
          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
          sudo chmod a+r /etc/apt/keyrings/docker.asc
          echo \
            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
            $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
            sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
          sudo apt-get update

          # upgrade Docker
          sudo apt install --upgrade docker-ce docker-compose-plugin=2.27.0\*

      - name: Debug info
        run: |
          docker version --format '{{.Server.Version}}'
          docker compose version --short

      - uses: actions/checkout@v4

      # login to pull images from Github registry
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Login to Docker Hub to increase rate limit on pulls
        # https://github.com/docker/login-action?tab=readme-ov-file#dockerhub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
        # worst case of this failing is running into Docker Hub rate limit
        continue-on-error: true

      - name: Download images from artifacts (for forked PR's or dependabot)
        # trigger only for forked PR's that don't have permissions to push to the container registry
        if: ${{ env.use_registry == 'false' }}
        uses: actions/download-artifact@v4
        with:
          path: images/

      - name: Load images from artifacts (for forked PR's or dependabot)
        # trigger only for forked PR's that don't have permissions to push to the container registry
        if: ${{ env.use_registry == 'false' }}
        run: find images/ -type f -name *.tar.gz -exec  sh -c 'gunzip --stdout "{}" | docker load' \;

      - name: Pull additional docker images
        if: ${{ env.use_registry == 'false' }}
        # build env includes all images, this will pull additional images not loaded from artifacts for
        # forked PR's/dependabot build
        run: make pull env=build pull_args=--ignore-buildable

      - name: Pull all docker images
        if: ${{ env.use_registry == 'true' }}
        # build env includes all images, this will pull additional images from the registry for non-forked PR builds
        # main and release builds
        run: make pull env=build

      - name: Start development environment
        # disable autoreload in CI as it makes workers unstable
        run: DEVSERVER_ARGS=--noreload make up env=develop

      - name: Run development environment tests
        run: make develop-tests

      - name: Collect Docker Compose logs
        if: always()
        run: make logs-all-dump env=develop > docker-compose.log

      - uses: test-summary/action@v2.3
        with:
          paths: test-results.xml
        if: always()
        continue-on-error: true

      - name: Failure log
        if: failure()
        # log last few lines in case of failure for quick debugging
        run: make docker-compose args="logs --tail=100" env=develop

      - name: Archive test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: Playwright development environment test results (screenshots, video)
          path: test-results/
          if-no-files-found: ignore

      - name: Archive test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: Development environment test Docker Compose Logs
          path: docker-compose.log
          if-no-files-found: ignore

  batch-integration-test:
    needs: [build-docker]

    runs-on: ubuntu-24.04

    env:
      # used in `compose.yaml` files to determine version of images to pull
      RELEASE: "${{ needs.build-docker.outputs.internetnl_version }}"
      PY_COLORS: "1"

    steps:
      - name: Install specific Docker/Compose versions that are known to work
        run: |
          # install Docker apt repository
          sudo apt update
          sudo apt-get install ca-certificates curl
          sudo install -m 0755 -d /etc/apt/keyrings
          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
          sudo chmod a+r /etc/apt/keyrings/docker.asc
          echo \
            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
            $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
            sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
          sudo apt-get update

          # upgrade Docker
          sudo apt install --upgrade docker-ce docker-compose-plugin=2.27.0\*

      - name: Debug info
        run: |
          docker version --format '{{.Server.Version}}'
          docker compose version --short

      - name: Enable ip6tables in Docker
        run: |
          sudo bash -c 'echo "{ \"ip6tables\": true, \"experimental\":true}" > /etc/docker/daemon.json'
          sudo systemctl restart docker.service
          sudo ip6tables -I DOCKER-USER --dst ff00::/8 -j ACCEPT

      - uses: actions/checkout@v4

      # login to pull images from Github registry
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Login to Docker Hub to increase rate limit on pulls
        # https://github.com/docker/login-action?tab=readme-ov-file#dockerhub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
        # worst case of this failing is running into Docker Hub rate limit
        continue-on-error: true

      - name: Download images from artifacts (for forked PR's or dependabot)
        # trigger only for forked PR's that don't have permissions to push to the container registry
        if: ${{ env.use_registry == 'false' }}
        uses: actions/download-artifact@v4
        with:
          path: images/

      - name: Load images from artifacts (for forked PR's or dependabot)
        # trigger only for forked PR's that don't have permissions to push to the container registry
        if: ${{ env.use_registry == 'false' }}
        run: find images/ -type f -name *.tar.gz -exec  sh -c 'gunzip --stdout "{}" | docker load' \;

      - name: Pull additional docker images
        if: ${{ env.use_registry == 'false' }}
        # build env includes all images, this will pull additional images not loaded from artifacts for
        # forked PR's/dependabot build
        run: make pull env=build pull_args=--ignore-buildable

      - name: Pull all docker images
        if: ${{ env.use_registry == 'true' }}
        # build env includes all images, this will pull additional images from the registry for non-forked PR builds
        # main and release builds
        run: make pull env=build

      - name: Start test instance
        run: make up env=batch-test

      - name: Run batch tests
        run: make batch-tests-verbose env=batch-test

      - name: Collect Docker Compose logs
        if: always()
        run: make logs-all-dump env=test > docker-compose.log

      - uses: test-summary/action@v2.3
        with:
          paths: test-results.xml
        if: always()
        continue-on-error: true

      - name: Failure log
        if: failure()
        # log last few lines in case of failure for quick debugging
        run: make docker-compose args="logs --tail=100" env=test

      - name: Archive test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: Playwright batch test results (screenshots, video) ${{ matrix.os }}
          path: test-results/
          if-no-files-found: ignore

      - name: Archive test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: Batch test Docker Compose Logs ${{ matrix.os }}
          path: docker-compose.log
          if-no-files-found: ignore