Container workflow overhaul

.github/workflows/build-and-remove-template.yml

@@ -0,0 +1,73 @@
+name: Build and Remove Docker Image
+# A workflow template that handles building+removing SPRAS containers.
+
+# Required inputs are:
+# - path: The path to a container's Dockerfile directory, relative to the repo root.
+# - container: The registry/container to be built, without a tag.
+# - always_build: A boolean indicating whether the action should condition the container build
+#                 on detected changes.
+# - context: The optional build context path, relative to the repo root.
+on:
+  workflow_call:
+    inputs:
+      path:
+        required: true
+        type: string
+      container:
+        required: true
+        type: string
+      always_build:
+        required: false
+        type: boolean
+        default: false
+      context:
+        required: false
+        type: string
+
+jobs:
+  build-and-remove-docker-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      # Given the input path to the directory containing definitions of the container,
+      # check whether anything at that path has been modified relative to master's head.
+      # If so, we'll set `env.changed=true` to trigger the build in a later step
+      - name: Check for changes
+        if: ${{ inputs.always_build == false }}
+        id: check_changes
+        run: |
+          if git fetch origin master && git diff --name-only origin/master | grep -q "$(basename ${{ inputs.path }})"; then
+            echo "Changes detected to ${{inputs.path}}."
+            echo "changed=true" >> $GITHUB_ENV
+          else
+            echo "No changes detected to ${{ inputs.path}}"
+            echo "changed=false" >> $GITHUB_ENV
+          fi
+
+      - name: Toggle container build if 'always_build'
+        if: ${{ inputs.always_build }}
+        run: |
+          echo "Input for job 'always_build' is true -- building regardless of detected changes."
+          echo "changed=true" >> $GITHUB_ENV
+
+      - name: Build Docker image
+        if: env.changed == 'true'
+        uses: docker/build-push-action@v6
+        with:
+          context: ${{ inputs.context || inputs.path }}/.
+          file: ${{ inputs.path }}/Dockerfile
+          tags: ${{ inputs.container }}:latest
+          # The cache will always use the `latest` tag, as we assume that results in the highest
+          # amount of reusable container stuff.
+          cache-from: type=registry,ref=${{ inputs.container }}:latest
+          push: false
+
+      # Clean/remove the container as we go -- we don't actually need it around, and it
+      # counts against our action's disk quota.
+      # Here we use `|| true` to prevent the job from failing if the image doesn't exist or
+      # can't be removed for some reason
+      - name: Remove Docker image
+        if: env.changed == 'true'
+        run: docker rmi ${{ inputs.container }}:latest || true

.github/workflows/build-containers.yml

@@ -0,0 +1,60 @@
+name: Build SPRAS Containers
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  push:
+    branches:
+      - master
+
+jobs:
+  build-and-remove-omics1:
+    uses: "./.github/workflows/build-and-remove-template.yml"
+    with:
+      path: docker-wrappers/OmicsIntegrator1
+      container: reedcompbio/omics-integrator-1
+  build-and-remove-omics2:
+    uses: "./.github/workflows/build-and-remove-template.yml"
+    with:
+      path: docker-wrappers/OmicsIntegrator2
+      container: reedcompbio/omics-integrator-2
+  build-and-remove-pathlinker:
+    uses: "./.github/workflows/build-and-remove-template.yml"
+    with:
+      path: docker-wrappers/PathLinker
+      container: reedcompbio/pathlinker
+  build-and-remove-meo:
+    uses: "./.github/workflows/build-and-remove-template.yml"
+    with:
+      path: docker-wrappers/MEO
+      container: reedcompbio/meo
+  build-and-remove-mincostflow:
+    uses: "./.github/workflows/build-and-remove-template.yml"
+    with:
+      path: docker-wrappers/MinCostFlow
+      container: reedcompbio/mincostflow
+  build-and-remove-allpairs:
+    uses: "./.github/workflows/build-and-remove-template.yml"
+    with:
+      path: docker-wrappers/AllPairs
+      container: reedcompbio/allpairs
+  build-and-remove-domino:
+    uses: "./.github/workflows/build-and-remove-template.yml"
+    with:
+      path: docker-wrappers/DOMINO
+      container: reedcompbio/domino
+  build-and-remove-cytoscape:
+    uses: "./.github/workflows/build-and-remove-template.yml"
+    with:
+      path: docker-wrappers/Cytoscape
+      container: reedcompbio/py4cytoscape
+  build-and-remove-spras:
+    uses: "./.github/workflows/build-and-remove-template.yml"
+    with:
+      path: docker-wrappers/SPRAS
+      container: reedcompbio/spras
+      # Since any change to the SPRAS codebase would constitute a change to
+      # the container we produce in this step, build the container regardless
+      # of detected changes.
+      always_build: true
+      context: ./

.github/workflows/test-spras.yml

@@ -61,141 +61,6 @@ jobs:
       shell: bash --login {0}
       run: snakemake --cores 2 --configfile config/config.yaml --show-failed-logs
 
-  # Builds the Docker images
-  docker:
-    name: Build Docker images
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      # Pull from Docker Hub to use the cache
-      # https://medium.com/mobileforgood/coding-tips-patterns-for-continuous-integration-with-docker-on-travis-ci-9cedb8348a62
-      # https://github.com/docker/build-push-action/issues/7
-    - name: Pull Docker images
-      run: |
-        docker pull reedcompbio/omics-integrator-1:latest
-        docker pull reedcompbio/omics-integrator-2:v2
-        docker pull reedcompbio/pathlinker:v2
-        docker pull reedcompbio/meo:latest
-        docker pull reedcompbio/mincostflow:latest
-        docker pull reedcompbio/allpairs:v2
-        docker pull reedcompbio/domino:latest
-        docker pull reedcompbio/py4cytoscape:v3
-        docker pull reedcompbio/spras:v0.1.0
-    - name: Build Omics Integrator 1 Docker image
-      uses: docker/build-push-action@v1
-      with:
-        path: docker-wrappers/OmicsIntegrator1/.
-        dockerfile: docker-wrappers/OmicsIntegrator1/Dockerfile
-        repository: reedcompbio/omics-integrator-1
-        tags: latest
-        cache_froms: reedcompbio/omics-integrator-1:latest
-        push: false
-    - name: Remove Omics Integrator 1 Docker image
-      # Remove the image to prevent the cache from being used. Here we use
-      # `|| true` to prevent the job from failing if the image doesn't exist or
-      # can't be removed for some reason
-      run: docker rmi reedcompbio/omics-integrator-1:latest || true
-
-    - name: Build Omics Integrator 2 Docker image
-      uses: docker/build-push-action@v1
-      with:
-        path: docker-wrappers/OmicsIntegrator2/.
-        dockerfile: docker-wrappers/OmicsIntegrator2/Dockerfile
-        repository: reedcompbio/omics-integrator-2
-        tags: v2
-        cache_froms: reedcompbio/omics-integrator-2:latest
-        push: false
-    - name: Remove Omics Integrator 2 Docker image
-      run: docker rmi reedcompbio/omics-integrator-2:latest || true
-
-    - name: Build PathLinker Docker image
-      uses: docker/build-push-action@v1
-      with:
-        path: docker-wrappers/PathLinker/.
-        dockerfile: docker-wrappers/PathLinker/Dockerfile
-        repository: reedcompbio/pathlinker
-        tags: v2
-        cache_froms: reedcompbio/pathlinker:latest
-        push: false
-    - name: Remove PathLinker Docker image
-      run: docker rmi reedcompbio/pathlinker:latest || true
-
-    - name: Build Maximum Edge Orientation Docker image
-      uses: docker/build-push-action@v1
-      with:
-        path: docker-wrappers/MEO/.
-        dockerfile: docker-wrappers/MEO/Dockerfile
-        repository: reedcompbio/meo
-        tags: latest
-        cache_froms: reedcompbio/meo:latest
-        push: false
-    - name: Remove MEO Docker image
-      run: docker rmi reedcompbio/meo:latest || true
-
-    - name: Build MinCostFlow Docker image
-      uses: docker/build-push-action@v1
-      with:
-        path: docker-wrappers/MinCostFlow/.
-        dockerfile: docker-wrappers/MinCostFlow/Dockerfile
-        repository: reedcompbio/mincostflow
-        tags: latest
-        cache_froms: reedcompbio/mincostflow:latest
-        push: false
-    - name: Remove MinCostFlow Docker image
-      run: docker rmi reedcompbio/mincostflow:latest || true
-
-    - name: Build All Pairs Shortest Paths Docker image
-      uses: docker/build-push-action@v1
-      with:
-        path: docker-wrappers/AllPairs/.
-        dockerfile: docker-wrappers/AllPairs/Dockerfile
-        repository: reedcompbio/allpairs
-        tags: v2
-        cache_froms: reedcompbio/allpairs:latest
-        push: false
-    - name: Remove All Pairs Shortest Paths Docker image
-      run: docker rmi reedcompbio/allpairs:latest || true
-
-    - name: Build DOMINO Docker image
-      uses: docker/build-push-action@v1
-      with:
-        path: docker-wrappers/DOMINO/.
-        dockerfile: docker-wrappers/DOMINO/Dockerfile
-        repository: reedcompbio/domino
-        tags: latest
-        cache_froms: reedcompbio/domino:latest
-        push: false
-    - name: Remove DOMINO Docker image
-      run: docker rmi reedcompbio/domino:latest || true
-
-    - name: Build Cytoscape Docker image
-      uses: docker/build-push-action@v1
-      with:
-        path: docker-wrappers/Cytoscape/.
-        dockerfile: docker-wrappers/Cytoscape/Dockerfile
-        repository: reedcompbio/py4cytoscape
-        tags: v3
-        cache_froms: reedcompbio/py4cytoscape:v3
-        push: false
-    - name: Remove Cytoscape Docker image
-      run: docker rmi reedcompbio/py4cytoscape:v3 || true
-
-    - name: Build SPRAS Docker image
-      uses: docker/build-push-action@v1
-      with:
-        path: .
-        dockerfile: docker-wrappers/SPRAS/Dockerfile
-        repository: reedcompbio/spras
-        tags: v0.2.0
-        cache_froms: reedcompbio/spras:v0.2.0
-        push: false
-    - name: Remove SPRAS Docker image
-      run: docker rmi reedcompbio/spras:v0.2.0 || true
-
   # Run pre-commit checks on source files
   pre-commit:
     name: Run pre-commit checks

CONTRIBUTING.md

@@ -224,7 +224,7 @@ The pull request will be closed so that the `master` branch of the fork stays sy
 1. Document the usage of the Docker wrapper and the assumptions made when implementing the wrapper
 1. Add example usage for the new algorithm and its parameters to the template config file
 1. Write test functions and provide example input data in a new test subdirectory `test/<algorithm>`. Provide example data and algorithm/expected files names to lists or dicts in `test/generate-inputs` and `test/parse-outputs`. Use the full path with the names of the test files.
-1. Extend `.github/workflows/test-spras.yml` to pull and build the new Docker image
+1. Extend `.github/workflows/build-containers.yml` to pull and build the new Docker image
 
 When adding new algorithms, there are many other considerations that are not relevant with the simple Local Neighborhood example.
 Most algorithms require dependencies that need to be installed in the `Dockerfile`.