run_tests.yml

#
# BSD 2-Clause License
#
# Copyright (c) 2021-2024, Hewlett Packard Enterprise
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

name: run-tests

on:
  pull_request:
  push:
    branches:
      - develop

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  HOMEBREW_NO_ANALYTICS: "ON" # Make Homebrew installation a little quicker
  HOMEBREW_NO_AUTO_UPDATE: "ON"
  HOMEBREW_NO_BOTTLE_SOURCE_FALLBACK: "ON"
  HOMEBREW_NO_GITHUB_API: "ON"
  HOMEBREW_NO_INSTALL_CLEANUP: "ON"
  DEBIAN_FRONTEND: "noninteractive" # Disable interactive apt install sessions
  GIT_CLONE_PROTECTION_ACTIVE: false

jobs:
  run_tests:
    name: Run tests ${{ matrix.subset }} with ${{ matrix.os }}, Python ${{ matrix.py_v}}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        subset: [backends, slow_tests, group_a, group_b, dragon]
        os: [macos-12, macos-14, ubuntu-22.04] # Operating systems
        compiler: [8] # GNU compiler version
        rai: [1.2.7] # Redis AI versions
        py_v: ["3.9", "3.10", "3.11"] # Python versions
        exclude:
          - os: macos-14
            py_v: "3.9"

    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.py_v }}

      - name: Check Test Files are Marked
        run: |
          diff <(find tests -path tests/_legacy -prune -o -type f -name 'test_*.py' -print \
                 | xargs grep -l 'pytestmark' \
                 | sort) \
               <(find tests -path tests/_legacy -prune -o -type f -name 'test_*.py' -print \
                 | sort)

      - name: Install build-essentials for Ubuntu
        if: contains( matrix.os, 'ubuntu' )
        run: |
          sudo add-apt-repository ppa:ubuntu-toolchain-r/test
          sudo apt-get update
          sudo apt-get install -y build-essential
          sudo apt-get install -y wget

      - name: Install GNU make for MacOS and set GITHUB_PATH
        if: contains( matrix.os, 'macos' )
        run: |
          brew install make || true
          echo "$(brew --prefix)/opt/make/libexec/gnubin" >> $GITHUB_PATH

      - name: Build Singularity from source
        if: contains( matrix.os, 'ubuntu' ) && matrix.py_v == 3.9
        run: |
          sudo apt-get install -y libseccomp-dev pkg-config squashfs-tools cryptsetup curl git # wget build-essential
          echo 'export PATH=/usr/local/go/bin:$PATH' >> ~/.bashrc
          source ~/.bashrc
          export VERSION=1.0.0 # Apptainer (singularity) version
          wget https://github.com/apptainer/apptainer/releases/download/v${VERSION}/apptainer-${VERSION}.tar.gz
          tar -xzf apptainer-${VERSION}.tar.gz
          cd apptainer-${VERSION}
          ./mconfig
          make -C builddir
          sudo make -C builddir install

      - name: singularity pull test container # This lets us time how long the pull takes
        if: contains( matrix.os, 'ubuntu' ) && matrix.py_v == 3.9
        run: singularity pull docker://alrigazzi/smartsim-testing

      # Note: The develop branch of smartredis is installed first to ensure that any tests that depend
      # on developments of the client are brought in.
      - name: Install SmartSim (with ML backends)
        run: |
          python -m pip install git+https://github.com/CrayLabs/SmartRedis.git@develop#egg=smartredis
          python -m pip install .[dev,mypy]

      - name: Install ML Runtimes
        if: matrix.subset != 'dragon'
        run: smart build --device cpu -v

     
      - name: Install ML Runtimes (with dragon)
        if: matrix.subset == 'dragon'
        env:
          SMARTSIM_DRAGON_TOKEN: ${{ secrets.DRAGON_TOKEN }}
        run: |
          if [ -n "${SMARTSIM_DRAGON_TOKEN}" ]; then
            smart build --device cpu -v --dragon-repo dragonhpc/dragon-nightly --dragon-version 0.10
          else
            smart build --device cpu -v --dragon
          fi
          SP=$(python -c 'import site; print(site.getsitepackages()[0])')/smartsim/_core/config/dragon/.env
          LLP=$(cat $SP | grep LD_LIBRARY_PATH | awk '{split($0, array, "="); print array[2]}')
          echo "LD_LIBRARY_PATH=$LLP:$LD_LIBRARY_PATH" >> $GITHUB_ENV

      - name: Run mypy
        run: |
          make check-mypy

      # TODO: Re-enable static analysis once API is firmed up
      # - name: Run Pylint
      #   run: make check-lint

      # Run isort/black style check
      - name: Run isort
        run: isort --check-only --profile black ./smartsim ./tests

      # Run isort/black style check
      - name: Run black
        run: |
          black --exclude smartsim/version.py --check ./smartsim ./tests

      # Run pytest (backends subdirectory)
      - name: Run Pytest
        if: (matrix.subset == 'backends')
        run: |
          echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV
          py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ ./tests/backends

      # Run pytest (dragon subtests)
      - name: Run Dragon Pytest
        if: (matrix.subset == 'dragon' && matrix.os == 'ubuntu-22.04')
        run: |
          echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV
          dragon -s py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ -m ${{ matrix.subset }} ./tests

      # Run pytest (test subsets)
      - name: Run Pytest
        if: (matrix.subset != 'backends' && matrix.subset != 'dragon') # if not running backend tests or dragon tests
        run: |
          echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV
          py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ -m ${{ matrix.subset }} ./tests

      # Upload artifacts on failure, ignoring binary files
      - name: Upload Artifact
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: test_artifact
          path: |
            tests/test_output
            !**/*.so
            !**/*.pb
            !**/*.pt
            !**/core
          retention-days: 5

      - name: Upload Pytest coverage to Codecov
        uses: codecov/codecov-action@v4.5.0
        with:
          fail_ci_if_error: false
          files: ./coverage.xml