Merge pull request #9 from Sage-Bionetworks-Workflows/cleanup

Cleanup codebase
Sage-Bionetworks-Workflows · Aug 24, 2023 · b9d5c21 · b9d5c21
2 parents e096616 + 338bf4b
commit b9d5c21
Show file tree

Hide file tree

Showing 27 changed files with 177 additions and 675 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -3,54 +3,54 @@ name: ci
 on:
   push:
     branches:
-     - 'main'
+      - "main"
     paths:
-     - 'docker/**'
-     - '.github/workflows/ci.yml'
+      - "docker/**"
+      - ".github/workflows/ci.yml"
 
 env:
   REGISTRY: ghcr.io
 
 jobs:
-      docker:
-        runs-on: ubuntu-latest
-        permissions:
-          contents: read
-          packages: write
-          id-token: write
-
-        defaults:
-          run:
-            working-directory: './docker'
-
-        steps:
-          - name: Checkout GitHub Action
-            uses: actions/checkout@v3
-
-          - name: Login to GitHub Container Registry (GHCR)
-            uses: docker/login-action@v2
-            with:
-              registry: ${{ env.REGISTRY }}
-              username: ${{ github.actor }}
-              password: ${{ secrets.GITHUB_TOKEN }}
-              
-          - name: Extract Docker metadata
-            id: metadata
-            uses: docker/metadata-action@v4
-            with:
-              images: ${{ env.REGISTRY }}/${{ github.repository }}
-              tags: |
-                type=semver,pattern={{version}}
-                type=semver,pattern={{major}}.{{minor}} 
-                type=semver,pattern={{major}}
-                type=ref,event=branch
-                type=sha
-                latest
-
-          - name: Build and push to GHCR
-            uses: docker/build-push-action@v4
-            with:
-              context: ./docker
-              push: true
-              tags: ${{ steps.metadata.outputs.tags }}
-              labels: ${{ steps.metadata.outputs.labels }}
+  docker:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+
+    defaults:
+      run:
+        working-directory: "./docker"
+
+    steps:
+      - name: Checkout GitHub Action
+        uses: actions/checkout@v3
+
+      - name: Login to GitHub Container Registry (GHCR)
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: metadata
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ github.repository }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}} 
+            type=semver,pattern={{major}}
+            type=ref,event=branch
+            type=sha
+            latest
+
+      - name: Build and push to GHCR
+        uses: docker/build-push-action@v4
+        with:
+          context: ./docker
+          push: true
+          tags: ${{ steps.metadata.outputs.tags }}
+          labels: ${{ steps.metadata.outputs.labels }}
diff --git a/.github/workflows/scan-images.yml b/.github/workflows/scan-images.yml
@@ -1,22 +1,22 @@
 name: Scan image
 
 on:
-    push:
-      branches: [ "main" ]
-    pull_request:
-      branches: [ "main" ]
-    schedule:
-      - cron: '32 12 * * 2' # every tuesday at 12:32 UTC time
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+  schedule:
+    - cron: "32 12 * * 2" # every tuesday at 12:32 UTC time
 
 env:
-    REGISTRY: ghcr.io
-    IMAGE_TAG: latest # scan just latest image
+  REGISTRY: ghcr.io
+  IMAGE_TAG: latest # scan just latest image
 
 jobs:
   trivy:
-    name:  Run Trivy vulnerability scanner
+    name: Run Trivy vulnerability scanner
     permissions:
-        security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
     runs-on: ubuntu-latest
 
     steps:
@@ -31,8 +31,8 @@ jobs:
         id: metadata
         uses: docker/metadata-action@v4
         with:
-            images: ${{ env.REGISTRY }}/${{ github.repository }}
-            tags: ${{ env.IMAGE_TAG }}
+          images: ${{ env.REGISTRY }}/${{ github.repository }}
+          tags: ${{ env.IMAGE_TAG }}
 
       - name: Pull the image
         run: |
@@ -43,7 +43,7 @@ jobs:
         with:
           image-ref: ${{ steps.metadata.outputs.tags }}
           ignore-unfixed: true
-          severity: 'CRITICAL,HIGH'
+          severity: "CRITICAL,HIGH"
           limit-severities-for-sarif: true
           format: sarif
           output: trivy-results.sarif

diff --git a/.gitignore b/.gitignore
@@ -3,4 +3,5 @@ work/
 .nextflow.log*
 test-outputs/
 flowchart*
-outputs/*
+outputs/*
+.DS_Store
diff --git a/README.md b/README.md
@@ -3,50 +3,68 @@
 A NextFlow pipeline to run image rendering process to generate resources for the [HTAN Portal](https://github.com/ncihtan/htan-portal).
 
 - Converts bioformats files into OME-TIFF
-- Generates a `story.json` file using [Auto-Minerva](https://github.com/jmuhlich/auto-minerva)
+- Sets thresholds for each channel and pepares 4-channel overlay groups using [Auto-Minerva](https://github.com/jmuhlich/auto-minerva)
 - Renders a Minerva story using [Minerva Author](https://github.com/labsyspharm/minerva-author)
-- If the `--all` parameter is set, renders a thumbnail, an autominerva story and gets the metadata
-- If the `--miniature` parameter is set, renders a thumbnail image using [Miniature](https://github.com/adamjtaylor/miniature)
-- `--he` assumes the channel is a brighfield microscopy image of H&E stained tissue and uses a fixed, unscaled `story.json` and a custom color legend
-- `--input` can be the path to an image (with `*` wildcards) or a csv manifest of cloud storage uris (one per line).
+- Renders a thumbnail image using [Miniature](https://github.com/adamjtaylor/miniature)
 
 A Docker container ([ghcr.io/sage-bionetworks-workflows/nf-artist](https://github.com/sage-bionetworks-workflows/nf-artist/pkgs/container/nf-artist)) is used to ensure reproducibility.
 
 ## Example usage
 
 ```
-nextflow run ghcr.io/sage-bionetworks-workflows/nf-artist --input_path <path-to-image> --outdir <output-directory> --all
+nextflow run ghcr.io/sage-bionetworks-workflows/nf-artist --input <path-to-samplesheet> --outdir <output-directory>
+```
+
+
+## Output
+
+`nf-artist` outputs the following directory structure into the specified output directory (`outdir`):
+
+```
+├── outdir
+│   ├── <simpleName or id for first row of samplesheet>
+│   │   ├── thumbnail.jpeg
+│   │   ├── minerva
+│   │   │   ├── index.html
+│   │   │   ├── exhibit.json
+│   │   │   ├── story.json
+│   │   │   ├── Group-1
+│   │   │   │   ├── tile1.jpeg
+│   │   │   │   ├── ...
+│   │   │   ├── Group-<n>
+│   │   │   │   ├── tile1.jpeg
+│   │   │   │   ├── ...
+│   ├── < simpleName or id for n'th row of samplesheet>
 ```
 
 ## Options
 
-`--outdir` - output directory. Default: `.`  
-`--minerva`: Renders an [Auto-Minerva](https://github.com/jmuhlich/auto-minerva) story  
-`--miniature` - Renders a thumbnail image using [Miniature](https://github.com/adamjtaylor/miniature)   
-`--metadata` -  Extract headers from the image and save as a json  
-`--all` - set `--minerva` `--miniature` and `--metadata`   
-`--he` - Use an unscaled scene for Minerva story and thumbnail generation. Suitable for H&E images  
-`--input_csv` - Path to a csv with a file path, uid, or synapseID per row  
-`--input_synid` - A synapse ID  
-`--input_path` - The path to a file. Can take wildcards  
-`--watch_path` - A path to watch for files that are created or modified  
-`--watch_csv` - A path to a csv to watch for if it is modified  
-`--echo` - Echo outputs  
-`--keepBg` - Keep the background in thumbnails  
-`--level` - the pyramid level used in thumbnauls, Default: `-1` (highest)  
-`--bioformats2ometiff` - Convert images to ome-tiff. Default: `true`  
-`--synapseconfig` - Path to a synapseConfig file. Required for Synapse authentication  
+#### Input/Output Options:
+
+* **input**: Path to a CSV sample sheet. This parameter is required. (Type: String)
+
+* **outdir**: Specifies the directory where the output data should be saved. Default is "outputs". (Type: String)
+
+#### Miniature Options:
+
+* **remove_bg**: Setting this to true will remove the non-tissue background. Default is true. (Type: Boolean)
+
+* **level**: Specifies the pyramid level used in thumbnails. Default is -1 (smallest). (Type: Integer)
 
-## Example flow diagram:
+* **dimred**: The dimensionality reduction method used. Default is "umap". Options include "umap", "tsne", and "pca". (Type: String)
 
-![image](https://user-images.githubusercontent.com/14945787/133272620-18223615-ce22-41c3-807b-3f3007b8f080.png)
+* **colormap**: Specifies the colormap used. Ensure the colormap is compatible with the number of `n_components` selected. Default is "UCIE". 3D colormap options: "UCIE", "LAB", "RGB". 2D colormap options: "BREMM", "SCHUMANN", "STEIGER", "TEULING2", "ZIEGLER", "CUBEDIAGONAL". (Type: String)
 
-## Docker pointers
+* **n_components**: Specifies the number of components. Default is 3. Options are 2 and 3. (Type: Integer)
 
-### Test docker container
+#### Samplesheet requirements
 
-`docker run -ti ghcr.io/sage-bionetworks-workflows/nf-artist`
+The samplesheet specified in the `input` parameter should be a CSV file with the following columns
 
-## Build docker container
+- `image`: [string] Path or URI to image to be processed
+- `convert`: [boolean] Should the image be converted to a OME-TIFF
+- `he`: [boolean] Is the image a H&E image
+- `minerva`: [boolean] Should a Minerva story be generated
+- `miniatuee`: [boolean] Should a Miniature thumbnail be generated
+- `id`: *optional* [string] A custom identifier to replace image simpleName in output directory structure 
 
-`docker build -t ghcr.io/sage-bionetworks-workflows/nf-artist docker/`
diff --git a/TODO.md b/TODO.md
@@ -0,0 +1,14 @@
+# nf-artist
+nf-artist is a Nextflow pipeline to generate interactive multiplexed image explorations and thumbnails.
+
+- Add CI testing in Github action using included test data in `data/` and samplesheet `data/test_samplesheet.csv`
+- Move logic on if minerva story or miniature thumbnail are run out of samplesheet and into `params`
+
+## Minerva
+- ✅ ~~Ensure H&E images are rendered with a fixed legend~~
+- Allow for custom descriptions to be provided in the samplesheet and added into the Minerva story
+- Update to new version of Minerva with channel selection
+
+## Miniature
+
+- Split `make_miniature` into two processes, one to run the miniature script for multiplexed images and one (that can have lower resource requirements) that makes the brightfield thumbnail for H&E images with TiffSlide
diff --git a/assets/he_story.json b/assets/he_story.json
@@ -0,0 +1,21 @@
+{
+  "sample_info": {
+    "name": "",
+    "rotation": 0,
+    "text": "",
+    "pixels_per_micron": null
+  },
+  "defaults": [],
+  "groups": [
+    {
+      "label": "H&E",
+      "channels": [
+        { "color": "ffffff", "id": 0, "label": "H&E", "min": 0, "max": 1 }
+      ],
+      "render": [
+        { "color": "ffffff", "id": 0, "label": "H&E", "min": 0, "max": 1 }
+      ]
+    }
+  ],
+  "waypoints": []
+}
diff --git a/data/CMU-1-Small-Region.svs b/data/CMU-1-Small-Region.svs
diff --git a/data/cycif_tonsil_small.tiff b/data/cycif_tonsil_small.tiff
diff --git a/data/cycif_tonsil_very_small.ome.tiff b/data/cycif_tonsil_very_small.ome.tiff
diff --git a/data/exemplar-001_small.tif b/data/exemplar-001_small.tif
diff --git a/data/test.csv b/data/test.csv
diff --git a/data/test_samplesheet.csv b/data/test_samplesheet.csv
@@ -0,0 +1,3 @@
+image,convert,he,minerva,miniature,id
+datas/exemplar-001_small.tif,true,false,true,true,cycif
+data/CMU-1-Small-Region.svs,true,true,true,true,h_and_e
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -24,7 +24,6 @@ RUN conda-pack -n artist -o /tmp/env.tar && \
 # so now fix up paths:
 RUN /venv/bin/conda-unpack
 
-
 # The runtime-stage image; we can use Debian as the
 # base image since the Conda env also includes Python
 # for us.
@@ -48,14 +47,9 @@ RUN apt-get update && apt-get install --yes --no-install-recommends \
     libblosc1
 
 RUN source /venv/bin/activate && \
-    #git clone https://github.com/adamjtaylor/miniature.git && \
     git clone https://github.com/adamjtaylor/miniature.git -b v2 && \
-    #git clone https://github.com/jmuhlich/auto-minerva.git && \
     git clone https://github.com/adamjtaylor/auto-minerva.git -b minerva1point5 && \
-    git clone https://github.com/labsyspharm/minerva-author.git minerva-author-dev && \
-    git clone https://github.com/labsyspharm/minerva-author.git -b v1.11.2 &&  \
-    git clone https://github.com/ncihtan/image-header-validation && \
-    wget https://gist.githubusercontent.com/adamjtaylor/964e206bf1e6f302f6e512082e953193/raw/0acdea0736a027a260a0bb598f619552ff106758/index.html && \
+    git clone https://github.com/labsyspharm/minerva-author.git -b v1.14.0 &&  \
     pip install git+https://github.com/labsyspharm/minerva-lib-python@master#egg=minerva-lib && \
     pip install openslide-python && \
     pip install opencv-python-headless \

diff --git a/docker/environment.yml b/docker/environment.yml
@@ -31,4 +31,4 @@ dependencies:
   - imagemagick
   - tiffslide=2.1.2
   - libvips
-  - estimagic
+  - estimagic
diff --git a/main.nf b/main.nf
@@ -12,12 +12,6 @@ params.dimred = "umap"
 params.colormap = "UCIE"
 params.n_components = 3
 
-// made these into params so that they can be accessed anywhere in the pipeline
-// can be added to profiles or other configuration later
-params.heStory = 'https://gist.githubusercontent.com/adamjtaylor/3494d806563d71c34c3ab45d75794dde/raw/d72e922bc8be3298ebe8717ad2b95eef26e0837b/unscaled.story.json'
-params.heScript = 'https://gist.githubusercontent.com/adamjtaylor/bbadf5aa4beef9aa1d1a50d76e2c5bec/raw/1f6e79ab94419e27988777343fa2c345a18c5b1b/fix_he_exhibit.py'
-params.minerva_description_script = 'https://gist.githubusercontent.com/adamjtaylor/e51873a801fee39f1f1efa978e2b5e44/raw/c03d0e09ec58e4c391f5ce4ca4183abca790f2a2/inject_description.py'
-
 include { ARTIST } from './workflows/artist.nf'
 
 workflow NF_ARTIST {

diff --git a/modules/autominerva_story.nf b/modules/autominerva_story.nf
@@ -1,23 +1,19 @@
 process autominerva_story {
+  tag {"$meta.id"}
+  label "process_low"
   input:
       tuple val(meta), file(image) 
   output:
       tuple val(meta), file(image), file('story.json')
   publishDir "$params.outdir/$workflow.runName",
-    pattern: 'story.json',
-    saveAs: {filename -> "${meta.id}/$workflow.runName/story.json"}
+      pattern: 'story.json',
+      saveAs: {filename -> "${meta.id}/$workflow.runName/story.json"}
   stub: 
   """
   touch story.json
   """
   script:
-  if (meta.he) {
-    """
-    wget -O story.json $params.heStory
-    """
-  } else {
-    """
-    python3 /auto-minerva/story.py $image > 'story.json'
-    """
-  }
+  """
+  python3 /auto-minerva/story.py $image > 'story.json'
+  """
 }
diff --git a/modules/bioformats2ometiff.nf b/modules/bioformats2ometiff.nf
@@ -1,4 +1,5 @@
 process bioformats2ometiff {
+  tag {"$meta.id"}
   label "process_medium"
   input:
       tuple val(meta), file(image) 

diff --git a/modules/make_miniature.nf b/modules/make_miniature.nf
@@ -1,5 +1,6 @@
 process make_miniature {
-  label "process_medium"
+  tag {"$meta.id"}
+  label "process_high"
   input:
       tuple val(meta), file(image) 
   output: