test.yml

name: Run tests
on:
  push:
    branches: [main]
  pull_request:
    branches: [main, develop]
  merge_group:
    types: [checks_requested]
    branches: [main]

# Cancel if a newer run is started
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
  NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

jobs:
  pre-commit:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
      - uses: pre-commit/action@v3.0.0
        # FIXME Flip this off once we get to less than a couple hundred. Adding
        # this so it will only run against changed files. It'll make it much
        # easier to fix these as they come up rather than everything at once.
        with:
          extra_args: ""

  prettier:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - name: Install NodeJS
        uses: actions/setup-node@v4
        with:
          node-version: "20"

      - name: Install Prettier
        run: npm install -g prettier

      - name: Run Prettier --check
        run: prettier --check .

  editorconfig:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v4
        with:
          node-version: "20"

      - name: Install editorconfig-checker
        run: npm install -g editorconfig-checker

      - name: Run ECLint check
        run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test)

  pytest-changes:
    name: pytest-changes
    runs-on: ubuntu-latest
    outputs:
      # Expose matched filters as job 'modules' output variable
      modules: ${{ steps.filter.outputs.changes }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2 # To retrieve the preceding commit.

      # TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented
      - uses: mirpedrol/paths-filter@main
        id: filter
        with:
          filters: "tests/config/pytest_modules.yml"
          token: ""

  nf-test-changes:
    name: nf-test-changes
    runs-on: ubuntu-latest
    outputs:
      # Expose matched filters as job 'modules' output variable
      modules: ${{ steps.filter.outputs.changes }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2 # To retrieve the preceding commit.

      - name: Combine all tags.yml files
        id: get_tags
        run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml

      - name: debug
        run: cat .github/tags.yml

      # TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented
      - uses: mirpedrol/paths-filter@main
        id: filter
        with:
          filters: ".github/tags.yml"
          token: ""

  nf-core-lint:
    runs-on: ubuntu-latest
    name: nf-core-lint
    needs: [pytest-changes, nf-test-changes]
    if: ${{ (needs.pytest-changes.outputs.modules != '[]') || (needs.nf-test-changes.outputs.modules != '[]') }}
    strategy:
      fail-fast: false
      matrix:
        tags:
          [
            "${{ fromJson(needs.pytest-changes.outputs.modules) }}",
            "${{ fromJson(needs.nf-test-changes.outputs.modules) }}",
          ]
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"

      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - name: Install pip
        run: python -m pip install --upgrade pip

      - uses: actions/setup-java@v3
        with:
          distribution: "temurin"
          java-version: "17"
      - name: Setup Nextflow
        uses: nf-core/setup-nextflow@v1

      - name: Install nf-core tools development version
        run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev

      - name: Lint module ${{ matrix.tags }}
        run: nf-core modules lint ${{ matrix.tags }}
        if: ${{ !startsWith(matrix.tags, 'subworkflows/') }}

      - name: Remove substring from matrix.tags
        id: remove_substring
        run: echo subworkflow_names=$(echo "${{ matrix.tags }}" | sed 's/subworkflows\///g') >> $GITHUB_OUTPUT

      - name: Lint subworkflow ${{ matrix.tags }}
        run: nf-core subworkflows lint ${{steps.remove_substring.outputs.subworkflow_names}}
        if: ${{ startsWith(matrix.tags, 'subworkflows/') }}

  verify-markdown-file:
    runs-on: ubuntu-latest
    name: verify-markdown-file
    if: ${{ (needs.pytest-changes.outputs.modules != '[]') || (needs.nf-test-changes.outputs.modules != '[]') }}
    needs: [pytest-changes, nf-test-changes]
    strategy:
      fail-fast: false
      matrix:
        tags:
          [
            "${{ fromJson(needs.pytest-changes.outputs.modules) }}",
            "${{ fromJson(needs.nf-test-changes.outputs.modules) }}",
          ]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2 # To retrieve the preceding commit.
      - name: setup python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"
      - name: Download convertor from yml to md
        run: |
          curl -o ${{ github.workspace }}/yaml_to_md.py https://raw.githubusercontent.com/mskcc-omics-workflows/yaml_to_md/0.0.1/yaml_to_md.py
          curl -o ${{ github.workspace }}/requirements.txt https://raw.githubusercontent.com/mskcc-omics-workflows/yaml_to_md/0.0.1/requirements.txt
      - name: Give convertor permissions
        run: chmod +x yaml_to_md.py
      - name: Install python packages
        run: |
          python -m pip install --upgrade pip
          pip install -r ${{ github.workspace }}/requirements.txt
      - name: Update ${{ matrix.tags }}
        run: |
          MATRIX_FRAGMENT="${{ matrix.tags }}"
          TEMP_NAME=$(echo $MATRIX_FRAGMENT | sed 's/subworkflows\///g')
          SW_NAME=$(echo $TEMP_NAME | sed 's/modules\///g')
          echo "SW_NAME=$SW_NAME" >> $GITHUB_ENV
          echo $SW_NAME
      - uses: jaywcjlove/github-action-read-file@main
        id: cat_file_module
        with:
          branch: ${{ github.ref }}
          path: modules/msk/${{ env.SW_NAME }}/meta.yml
      - run: echo "${{ steps.cat_file_module.outputs.size }}"
      - name: Write Module yaml to temp
        if: ${{ steps.cat_file_module.outputs.size }}
        id: write_yml_module
        run: |
          echo '${{ steps.cat_file_module.outputs.content }}' > ${{ github.workspace }}/temp.yml
          FEATURE_TYPE="modules"
          echo "FEATURE_TYPE=$FEATURE_TYPE" >> $GITHUB_ENV
      - uses: jaywcjlove/github-action-read-file@main
        if: ${{ steps.cat_file_module.outputs.size == '' }}
        id: cat_file_subworkflow
        with:
          branch: ${{ github.ref }}
          path: subworkflows/msk/${{ env.SW_NAME }}/meta.yml
      - name: Write Subworkflow yaml to temp
        if: ${{ steps.cat_file_subworkflow.outputs.size }}
        id: write_yml_subworkflow
        run: |
          echo '${{ steps.cat_file_subworkflow.outputs.content }}' > ${{ github.workspace }}/temp.yml
          FEATURE_TYPE="subworkflows"
          echo "FEATURE_TYPE=$FEATURE_TYPE" >> $GITHUB_ENV
      - name: Run convertor to verify yaml file
        run: |
          echo ${{ matrix.tags }}
          echo ${{ env.FEATURE_TYPE }}
          python ${{ github.workspace }}/yaml_to_md.py validate --yaml-file ${{ github.workspace }}/temp.yml --schema-url https://raw.githubusercontent.com/mskcc-omics-workflows/yaml_to_md/0.0.1/nextflow_schema/${{ env.FEATURE_TYPE }}/meta-schema.json

  pytest:
    runs-on: ubuntu-latest
    name: pytest
    needs: [pytest-changes]
    if: needs.pytest-changes.outputs.modules != '[]'
    strategy:
      fail-fast: false
      matrix:
        tags: ["${{ fromJson(needs.pytest-changes.outputs.modules) }}"]
        profile: ["docker", "singularity", "conda"]
        exclude:
          - tags: "nf-test"
    env:
      NXF_ANSI_LOG: false
      SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"

      - name: Install Python dependencies
        run: python -m pip install --upgrade pip pytest-workflow cryptography

      - uses: actions/setup-java@v3
        with:
          distribution: "temurin"
          java-version: "17"
      - name: Setup Nextflow ${{ matrix.NXF_VER }}
        uses: nf-core/setup-nextflow@v1
        with:
          version: "${{ matrix.NXF_VER }}"

      - name: Setup apptainer
        if: matrix.profile == 'singularity'
        uses: eWaterCycle/setup-apptainer@main

      - name: Set up Singularity
        if: matrix.profile == 'singularity'
        run: |
          mkdir -p $NXF_SINGULARITY_CACHEDIR
          mkdir -p $NXF_SINGULARITY_LIBRARYDIR

      - name: Set up miniconda
        uses: conda-incubator/setup-miniconda@v2
        with:
          miniconda-version: "latest"
          channels: conda-forge,bioconda,defaults
          python-version: ${{ matrix.python-version }}

      - name: Conda setup
        run: |
          conda clean -a
          conda install -n base conda-libmamba-solver
          conda config --set solver libmamba
          echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
          echo $(realpath python) >> $GITHUB_PATH

      # Set up secrets
      - name: Set up nextflow secrets
        if: env.SENTIEON_LICENSE_BASE64 != null
        run: |
          nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }}
          nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }}
          SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d)
          SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d)
          SENTIEON_AUTH_DATA=$(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE")
          SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0)
          nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64

      # Test the module
      - name: Run pytest-workflow
        # only use one thread for pytest-workflow to avoid race condition on conda cache.
        run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes

      - name: Output log on failure
        if: failure()
        run: |
          sudo apt-get update > /dev/null
          sudo apt-get install bat > /dev/null
          batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err}

      - name: Setting global variables
        uses: actions/github-script@v6
        id: parsed
        with:
          script: |
            return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"')
          result-encoding: string

      - name: Upload logs on failure
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }}
          path: |
            /home/runner/pytest_workflow_*/*/.nextflow.log
            /home/runner/pytest_workflow_*/*/log.out
            /home/runner/pytest_workflow_*/*/log.err
            /home/runner/pytest_workflow_*/*/work
            !/home/runner/pytest_workflow_*/*/work/conda
            !/home/runner/pytest_workflow_*/*/work/singularity
            !${{ github.workspace }}/.singularity

  nf-test:
    runs-on: ubuntu-latest
    name: nf-test
    needs: [nf-test-changes]
    if: needs.nf-test-changes.outputs.modules != '[]'
    strategy:
      fail-fast: false
      matrix:
        tags: ["${{ fromJson(needs.nf-test-changes.outputs.modules) }}"]
        profile: ["docker", "singularity", "conda"]
        exclude:
          - tags: "nf-test"
          - profile: "conda"
            tags: gbcms
          - profile: "conda"
            tags: polysolver
          - profile: "conda"
            tags: snppileup
          - profile: "conda"
            tags: facets
          - profile: "conda"
            tags: mutect1
          - profile: "conda"
            tags: phylowgs/createinput
          - profile: "conda"
            tags: phylowgs/parsecnvs
          - profile: "conda"
            tags: phylowgs/multievolve
          - profile: "conda"
            tags: phylowgs/writeresults
          - profile: "conda"
            tags: netmhcpan
          - profile: "conda"
            tags: neoantigeninput
          - profile: "conda"
            tags: neoantigenediting/aligntoIEDB
          - profile: "conda"
            tags: neoantigenediting/computeFitness
          - profile: "conda"
            tags: subworkflows/neoantigen_editing
          - profile: "conda"
            tags: subworkflows/phylowgs
          - profile: "conda"
            tags: netmhcstabpan
          - profile: "conda"
            tags: ppflagfixer
          - profile: "conda"
            tags: neoantigenutils/generatemutfasta
          - profile: "conda"
            tags: neoantigenutils/formatnetmhcpan
          - profile: "conda"
            tags: neoantigenutils/generatehlastring
          - profile: "conda"
            tags: neoantigenutils/neoantigeninput
          - profile: "conda"
            tags: subworkflows/netmhcstabandpan
          - profile: "conda"
            tags: salmon/index
          - profile: "conda"
            tags: genomenexus/vcf2maf
          - profile: "conda"
            tags: genomenexus/annotationpipeline
          - profile: "conda"
            tags: ppflagfixer
          - profile: "conda"
            tags: pvmaf/tagtraceback
          - profile: "conda"
            tags: pvmaf/concat
          - profile: "conda"
            tags: genotypevariants/all
          - profile: "conda"
            tags: subworkflows/traceback
          - profile: "conda"
            tags: salmon/index
          - profile: "conda"
            tags: subworkflows/genome_nexus
          - profile: "conda"
            tags: salmon/quant
          - profile: "conda"
            tags: rediscoverte

    env:
      NXF_ANSI_LOG: false
      SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
      NFTEST_VER: "0.8.2"

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-java@v3
        with:
          distribution: "temurin"
          java-version: "17"
      - name: Setup Nextflow
        uses: nf-core/setup-nextflow@v1

      - name: Cache nf-test installation
        id: cache-software
        uses: actions/cache@v3
        with:
          path: |
            /usr/local/bin/nf-test
            /home/runner/.nf-test/nf-test.jar
          key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest

      - name: Install nf-test
        if: steps.cache-software.outputs.cache-hit != 'true'
        run: |
          wget -qO- https://code.askimed.com/install/nf-test | bash
          sudo mv nf-test /usr/local/bin/

      - name: Setup apptainer
        if: matrix.profile == 'singularity'
        uses: eWaterCycle/setup-apptainer@main

      - name: Set up Singularity
        if: matrix.profile == 'singularity'
        run: |
          mkdir -p $NXF_SINGULARITY_CACHEDIR
          mkdir -p $NXF_SINGULARITY_LIBRARYDIR

      - name: Set up miniconda
        uses: conda-incubator/setup-miniconda@v2
        with:
          miniconda-version: "latest"
          auto-update-conda: true
          channels: conda-forge,bioconda,defaults
          python-version: ${{ matrix.python-version }}

      - name: Conda setup
        run: |
          conda clean -a
          conda install -n base conda-libmamba-solver
          conda config --set solver libmamba
          echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
          echo $(realpath python) >> $GITHUB_PATH

      # Set up secrets
      - name: Set up nextflow secrets
        if: env.SENTIEON_LICENSE_BASE64 != null
        run: |
          nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }}
          nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }}
          SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d)
          SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d)
          SENTIEON_AUTH_DATA=$(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE")
          SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0)
          nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64

      # Test the module
      - name: Run nf-test
        run: |
          NXF_TEST_DIR=~ \
          nf-test test \
            --profile=${{ matrix.profile }} \
            --tag ${{ matrix.tags }} \
            --tap=test.tap \
            --verbose

      - uses: pcolby/tap-summary@v1
        with:
          path: >-
            test.tap

  confirm-pass:
    runs-on: ubuntu-latest
    needs:
      [
        prettier,
        editorconfig,
        pytest-changes,
        nf-core-lint,
        pytest,
        nf-test-changes,
        nf-test,
      ]
    if: always()
    steps:
      - name: All tests ok
        if: ${{ success() || !contains(needs.*.result, 'failure') }}
        run: exit 0
      - name: One or more tests failed
        if: ${{ contains(needs.*.result, 'failure') }}
        run: exit 1

      - name: debug-print
        if: always()
        run: |
          echo "toJSON(needs) = ${{ toJSON(needs) }}"
          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"