Wave with Docker and Spack

.github/workflows/test.yml

@@ -382,14 +382,17 @@ jobs:
         with:
           distribution: "temurin"
           java-version: "17"
-      - name: Setup Nextflow ${{ matrix.NXF_VER }}
+      - name: Setup Nextflow
         uses: nf-core/setup-nextflow@v1
         with:
-          version: "${{ matrix.NXF_VER }}"
+          version: latest-everything
 
-      - name: Setup apptainer
+      - name: Install Singularity
         if: matrix.profile == 'singularity'
-        uses: eWaterCycle/setup-apptainer@main
+        run: |
+          wget https://github.com/sylabs/singularity/releases/download/v4.0.2/singularity-ce_4.0.2-jammy_amd64.deb
+          sudo apt-get install -y libfuse2
+          sudo dpkg -i singularity-ce_4.0.2-jammy_amd64.deb
 
       - name: Set up Singularity
         if: matrix.profile == 'singularity'
@@ -580,6 +583,8 @@ jobs:
           java-version: "17"
       - name: Setup Nextflow
         uses: nf-core/setup-nextflow@v1
+        with:
+          version: latest-everything
 
       - name: Cache nf-test installation
         id: cache-software
@@ -596,9 +601,12 @@ jobs:
           wget -qO- https://code.askimed.com/install/nf-test | bash
           sudo mv nf-test /usr/local/bin/
 
-      - name: Setup apptainer
+      - name: Install Singularity
         if: matrix.profile == 'singularity'
-        uses: eWaterCycle/setup-apptainer@main
+        run: |
+          wget https://github.com/sylabs/singularity/releases/download/v4.0.2/singularity-ce_4.0.2-jammy_amd64.deb
+          sudo apt-get install -y libfuse2
+          sudo dpkg -i singularity-ce_4.0.2-jammy_amd64.deb
 
       - name: Set up Singularity
         if: matrix.profile == 'singularity'

.github/workflows/wave.yml

@@ -0,0 +1,167 @@
+name: Wave
+# When environment.yml is changed
+on:
+  pull_request:
+    paths:
+      - "**/environment.yml"
+      - "**/Dockerfile"
+      - "**/spack.yml"
+
+env:
+  WAVE_VER: "1.2.0"
+
+jobs:
+  gen-matrix:
+    name: generate-matrix
+    runs-on: ubuntu-latest
+    outputs:
+      conda-matrix: ${{ steps.conda-diff.outputs.all_changed_files }}
+      docker-matrix: ${{ steps.docker-diff.outputs.all_changed_files }}
+      spack-matrix: ${{ steps.spack-diff.outputs.all_changed_files }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Find conda differences
+        id: conda-diff
+        uses: tj-actions/changed-files@v42
+        with:
+          json: true
+          escape_json: false
+          files: |
+            modules/**/environment.yml
+
+      - name: Find Dockerfile differences
+        id: docker-diff
+        uses: tj-actions/changed-files@v42
+        with:
+          json: true
+          escape_json: false
+          files: |
+            modules/**/Dockerfile
+
+      - name: Find Spack differences
+        id: spack-diff
+        uses: tj-actions/changed-files@v42
+        with:
+          json: true
+          escape_json: false
+          files: |
+            modules/**/spack.yml
+
+  conda-build:
+    if: "${{ fromJson(needs.gen-matrix.outputs.conda-matrix) }}"
+    needs: gen-matrix
+    name: Conda
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        files: "${{ fromJson(needs.gen-matrix.outputs.conda-matrix) }}"
+    steps:
+      # Use the @nf-core-bot token to check out so we can push later
+      - uses: actions/checkout@v4
+
+      - name: Install wave-cli
+        run: |
+          wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64
+          sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave
+          chmod +x /usr/local/bin/wave
+
+      - name: Create a registry name
+        uses: actions/github-script@v7
+        id: registry-name
+        with:
+          result-encoding: string
+          script: |
+            return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/environment.yml', '').replace('/', '_');
+
+      - name: Build container
+        run: |
+          wave --conda-file "${{ matrix.files }}" \
+              --freeze \
+              --await \
+              --build-repo quay.io/nf-core/modules/${{steps.registry-name.outputs.result}} \
+              --cache-repository quay.io/nf-core/wave-cache \
+              --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \
+              --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}
+
+  docker-build:
+    if: "${{ fromJson(needs.gen-matrix.outputs.docker-matrix) }}"
+    needs: gen-matrix
+    name: Docker
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        files: "${{ fromJson(needs.gen-matrix.outputs.docker-matrix) }}"
+    steps:
+      # Use the @nf-core-bot token to check out so we can push later
+      - uses: actions/checkout@v4
+
+      - name: Install wave-cli
+        run: |
+          wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64
+          sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave
+          chmod +x /usr/local/bin/wave
+
+      - name: Create a registry name
+        uses: actions/github-script@v7
+        id: registry-name
+        with:
+          result-encoding: string
+          script: |
+            return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/Dockerfile', '').replace('/', '_');
+
+      - name: Build container
+        run: |
+          wave -f "${{ matrix.files }}" \
+              --freeze \
+              --await \
+              --build-repo quay.io/nf-core/modules/${{steps.registry-name.outputs.result}} \
+              --cache-repository quay.io/nf-core/wave-cache \
+              --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \
+              --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}
+
+  # FIXME https://github.com/seqeralabs/wave/issues/351
+  # spack-build:
+  #   if: "${{ fromJson(needs.gen-matrix.outputs.spack-matrix) }}"
+  #   needs: gen-matrix
+  #   name: Spack
+  #   runs-on: ubuntu-latest
+  #   timeout-minutes: 60
+  #   strategy:
+  #     fail-fast: false
+  #     max-parallel: 4
+  #     matrix:
+  #       files: "${{ fromJson(needs.gen-matrix.outputs.spack-matrix) }}"
+  #   steps:
+  #     # Use the @nf-core-bot token to check out so we can push later
+  #     - uses: actions/checkout@v4
+
+  #     - name: Install wave-cli
+  #       run: |
+  #         wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64
+  #         sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave
+  #         chmod +x /usr/local/bin/wave
+
+  #     - name: Create a registry name
+  #       uses: actions/github-script@v7
+  #       id: registry-name
+  #       with:
+  #         result-encoding: string
+  #         script: |
+  #           return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/spack.yml', '').replace('/', '_');
+
+  #     - name: Build container
+  #       run: |
+  #         wave --spack-file "${{ matrix.files }}" \
+  #             --freeze \
+  #             --await \
+  #             --build-repo quay.io/nf-core/modules/${{steps.registry-name.outputs.result}} \
+  #             --cache-repository quay.io/nf-core/wave-cache \
+  #             --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \
+  #             --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}

modules/nf-core/bclconvert/Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 # Dockerfile to create container with bcl-convert
-# Push to nfcore/bclconvert:<VER>
+# Push to quay.io/nf-core/modules/bclconvert:<VER>
 
 FROM debian:bullseye-slim
 ARG BCLCONVERT_VERSION="4.2.7"

modules/nf-core/bowtie/align/environment.yml

@@ -5,3 +5,4 @@ channels:
   - defaults
 dependencies:
   - bioconda::bowtie=1.3.0
+  - bioconda::samtools=1.16.1

modules/nf-core/bowtie/align/main.nf

@@ -3,9 +3,7 @@ process BOWTIE_ALIGN {
     label 'process_high'
 
     conda "${moduleDir}/environment.yml"
-    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/mulled-v2-ffbf83a6b0ab6ec567a336cf349b80637135bca3:c84c7c55c45af231883d9ff4fe706ac44c479c36-0' :
-        'biocontainers/mulled-v2-ffbf83a6b0ab6ec567a336cf349b80637135bca3:c84c7c55c45af231883d9ff4fe706ac44c479c36-0' }"
+    container "nf-core/modules/bowtie_align:bowtie_align--d797c0ccd2d330fd"
 
     input:
     tuple val(meta), path(reads)

modules/nf-core/bowtie/align/spack.yml

@@ -0,0 +1,7 @@
+spack:
+  specs:
+    - [email protected]
+    - [email protected]
+
+  concretizer:
+    unify: true

modules/nf-core/bowtie/build/environment.yml

@@ -5,3 +5,4 @@ channels:
   - defaults
 dependencies:
   - bioconda::bowtie=1.3.0
+  - bioconda::samtools=1.16.1

modules/nf-core/bowtie/build/main.nf

@@ -3,9 +3,7 @@ process BOWTIE_BUILD {
     label 'process_high'
 
     conda "${moduleDir}/environment.yml"
-    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/bowtie:1.3.0--py38hed8969a_1' :
-        'biocontainers/bowtie:1.3.0--py38hed8969a_1' }"
+    container "nf-core/modules/bowtie_build:bowtie_build--4ee5ca3f4bf65e6a"
 
     input:
     path fasta

modules/nf-core/gatk4/germlinecnvcaller/environment.yml

@@ -3,3 +3,7 @@ channels:
   - conda-forge
   - bioconda
   - defaults
+dependencies:
+  - bioconda::gatk4=4.5.0.0
+  - pip:
+      - https://swift.rc.nectar.org.au/v1/AUTH_377/public/gatkPythonPackageArchive.zip

modules/nf-core/samtools/view/environment.yml

@@ -4,5 +4,7 @@ channels:
   - bioconda
   - defaults
 dependencies:
+  # renovate: datasource=conda depName=bioconda/samtools
   - bioconda::samtools=1.19.2
+  # renovate: datasource=conda depName=bioconda/htslib
   - bioconda::htslib=1.19.1

modules/nf-core/samtools/view/main.nf

@@ -3,9 +3,7 @@ process SAMTOOLS_VIEW {
     label 'process_low'
 
     conda "${moduleDir}/environment.yml"
-    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/samtools:1.19.2--h50ea8bc_0' :
-        'biocontainers/samtools:1.19.2--h50ea8bc_0' }"
+    container "nf-core/modules/samtools_view:samtools_view--033f764f39eb1f2d"
 
     input:
     tuple val(meta), path(input), path(index)

tests/config/nextflow.config

@@ -14,6 +14,7 @@ process {
 if ("$PROFILE" == "singularity") {
     singularity.enabled = true
     singularity.autoMounts = true
+    singularity.ociAutoPull = true
 } else if ("$PROFILE" == "conda") {
     conda.enabled = true
 } else if ("$PROFILE" == "mamba") {
@@ -41,5 +42,5 @@ conda { createTimeout = "120 min" }
 includeConfig 'test_data.config'
 
 manifest {
-    nextflowVersion = '!>=23.04.0'
+    nextflowVersion = '!>=23.12.0-edge'
 }

tests/config/nf-test.config

@@ -15,6 +15,7 @@ profiles {
     singularity {
         singularity.enabled = true
         singularity.autoMounts = true
+        singularity.ociAutoPull = true
     }
     conda {
         conda.enabled = true
@@ -46,5 +47,5 @@ conda { createTimeout = "120 min" }
 includeConfig 'test_data.config'
 
 manifest {
-    nextflowVersion = '!>=23.04.0'
+    nextflowVersion = '!>=23.12.0-edge'
 }