Simplify output structure

.github/workflows/ci.yml

@@ -0,0 +1,56 @@
+name: ci
+
+on:
+  push:
+    branches:
+      - "main"
+    paths:
+      - "docker/**"
+      - ".github/workflows/ci.yml"
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+
+    defaults:
+      run:
+        working-directory: "./docker"
+
+    steps:
+      - name: Checkout GitHub Action
+        uses: actions/checkout@v3
+
+      - name: Login to GitHub Container Registry (GHCR)
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: metadata
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ github.repository }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}} 
+            type=semver,pattern={{major}}
+            type=ref,event=branch
+            type=sha
+            latest
+
+      - name: Build and push to GHCR
+        uses: docker/build-push-action@v4
+        with:
+          context: ./docker
+          push: true
+          tags: ${{ steps.metadata.outputs.tags }}
+          labels: ${{ steps.metadata.outputs.labels }}

.github/workflows/scan-images.yml

@@ -0,0 +1,72 @@
+name: Scan image
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+  schedule:
+    - cron: "32 12 * * 2" # every tuesday at 12:32 UTC time
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_TAG: latest # scan just latest image
+
+jobs:
+  trivy:
+    name: Run Trivy vulnerability scanner
+    permissions:
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: metadata
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ github.repository }}
+          tags: ${{ env.IMAGE_TAG }}
+
+      - name: Pull the image
+        run: |
+          docker pull ${{ steps.metadata.outputs.tags }}
+
+      - name: Run Trivy vulnerability scanner for any major issues
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ${{ steps.metadata.outputs.tags }}
+          ignore-unfixed: true
+          severity: "CRITICAL,HIGH"
+          limit-severities-for-sarif: true
+          format: sarif
+          output: trivy-results.sarif
+          timeout: 20m # trivy default timeout is 5m, we need longer for our image
+
+      # Show all detected issues.
+      - name: Run Trivy vulnerability scanner for local output
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ${{ steps.metadata.outputs.tags }}
+          format: table
+          timeout: 20m # trivy default timeout is 5m, we need longer for our image
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: trivy-results.sarif
+          category: ${{ steps.metadata.outputs.tags }} image
+          wait-for-processing: true
+
+      - name: Detain results for debug if needed
+        uses: actions/upload-artifact@v3
+        with:
+          name: trivy-results.sarif
+          path: trivy-results.sarif
+          if-no-files-found: error

.gitignore

@@ -1,4 +1,7 @@
 .nextflow/
 work/
 .nextflow.log*
-test-outputs/
+test-outputs/
+flowchart*
+outputs/*
+.DS_Store

README.md

@@ -1,52 +1,70 @@
 # HTAN Artist
 
-A NextFlow piepline to run image rendering process to generate resources for the [HTAN Portal](https://github.com/ncihtan/htan-portal).
+A NextFlow pipeline to run image rendering process to generate resources for the [HTAN Portal](https://github.com/ncihtan/htan-portal).
 
 - Converts bioformats files into OME-TIFF
-- Generates a `story.json` file using [Auto-Minerva](https://github.com/jmuhlich/auto-minerva)
+- Sets thresholds for each channel and pepares 4-channel overlay groups using [Auto-Minerva](https://github.com/jmuhlich/auto-minerva)
 - Renders a Minerva story using [Minerva Author](https://github.com/labsyspharm/minerva-author)
-- If the `--all` parameter is set, renders a thumbnail, an autominerva story and gets the metadata
-- If the `--miniature` parameter is set, renders a thumbnail image using [Miniature](https://github.com/adamjtaylor/miniature)
-- `--he` assumes the channel is a brighfield microscopy image of H&E stained tissue and uses a fixed, unscaled `story.json` and a custom color legend
-- `--input` can be the path to an image (with `*` wildcards) or a csv manifest of cloud storage uris (one per line).
+- Renders a thumbnail image using [Miniature](https://github.com/adamjtaylor/miniature)
 
-A Docker container ([adamjtaylor/htan-artist](https://hub.docker.com/repository/docker/adamjtaylor/htan-artist)) is used to ensure reproducibility.
+A Docker container ([ghcr.io/sage-bionetworks-workflows/nf-artist](https://github.com/sage-bionetworks-workflows/nf-artist/pkgs/container/nf-artist)) is used to ensure reproducibility.
 
 ## Example usage
 
 ```
-nextflow run adamjtaylor/htan-artist --input_path <path-to-image> --outdir <output-directory> --all
+nextflow run Sage-Bionetworks-Workflows/nf-artist --input <path-to-samplesheet> --outdir <output-directory>
+```
+
+
+## Output
+
+`nf-artist` outputs the following directory structure into the specified output directory (`outdir`):
+
+```
+├── outdir
+│   ├── <simpleName or id for first row of samplesheet>
+│   │   ├── thumbnail.jpeg
+│   │   ├── minerva
+│   │   │   ├── index.html
+│   │   │   ├── exhibit.json
+│   │   │   ├── story.json
+│   │   │   ├── Group-1
+│   │   │   │   ├── tile1.jpeg
+│   │   │   │   ├── ...
+│   │   │   ├── Group-<n>
+│   │   │   │   ├── tile1.jpeg
+│   │   │   │   ├── ...
+│   ├── < simpleName or id for n'th row of samplesheet>
 ```
 
 ## Options
 
-`--outdir` - output directory. Default: `.`  
-`--minerva`: Renders an [Auto-Minerva](https://github.com/jmuhlich/auto-minerva) story  
-`--miniature` - Renders a thumbnail image using [Miniature](https://github.com/adamjtaylor/miniature)   
-`--metadata` -  Extract headers from the image and save as a json  
-`--all` - set `--minerva` `--miniature` and `--metadata`   
-`--he` - Use an unscaled scene for Minerva story and thumbnail generation. Suitable for H&E images  
-`--input_csv` - Path to a csv with a file path, uid, or synapseID per row  
-`--input_synid` - A synapse ID  
-`--input_path` - The path to a file. Can take wildcards  
-`--watch_path` - A path to watch for files that are created or modified  
-`--watch_csv` - A path to a csv to watch for if it is modified  
-`--echo` - Echo outputs  
-`--keepBg` - Keep the background in thumbnails  
-`--level` - the pyramid level used in thumbnauls, Default: `-1` (highest)  
-`--bioformats2ometiff` - Convert images to ome-tiff. Default: `true`  
-`--synapseconfig` - Path to a synapseConfig file. Required for Synapse authentication  
+#### Input/Output Options:
+
+* **input**: Path to a CSV sample sheet. This parameter is required. (Type: String)
+
+* **outdir**: Specifies the directory where the output data should be saved. Default is "outputs". (Type: String)
+
+#### Miniature Options:
+
+* **remove_bg**: Setting this to true will remove the non-tissue background. Default is true. (Type: Boolean)
+
+* **level**: Specifies the pyramid level used in thumbnails. Default is -1 (smallest). (Type: Integer)
 
-## Example flow diagram:
+* **dimred**: The dimensionality reduction method used. Default is "umap". Options include "umap", "tsne", and "pca". (Type: String)
 
-![image](https://user-images.githubusercontent.com/14945787/133272620-18223615-ce22-41c3-807b-3f3007b8f080.png)
+* **colormap**: Specifies the colormap used. Ensure the colormap is compatible with the number of `n_components` selected. Default is "UCIE". 3D colormap options: "UCIE", "LAB", "RGB". 2D colormap options: "BREMM", "SCHUMANN", "STEIGER", "TEULING2", "ZIEGLER", "CUBEDIAGONAL". (Type: String)
 
-## Docker pointers
+* **n_components**: Specifies the number of components. Default is 3. Options are 2 and 3. (Type: Integer)
 
-### Test docker container
+#### Samplesheet requirements
 
-`docker run -ti adamjtaylor/htan-artist`
+The samplesheet specified in the `input` parameter should be a CSV file with the following columns
 
-## Build docker container
+- `image`: [string] Path or URI to image to be processed
+- `convert`: [boolean] Should the image be converted to a OME-TIFF
+- `he`: [boolean] Is the image a H&E image
+- `minerva`: [boolean] Should a Minerva story be generated
+- `miniatuee`: [boolean] Should a Miniature thumbnail be generated
+- `id`: *optional* [string] A custom identifier to replace image simpleName in output directory structure 
 
-`docker build -t adamjtaylor/htan-artist docker/`

TODO.md

@@ -0,0 +1,14 @@
+# nf-artist
+nf-artist is a Nextflow pipeline to generate interactive multiplexed image explorations and thumbnails.
+
+- Add CI testing in Github action using included test data in `data/` and samplesheet `data/test_samplesheet.csv`
+- Move logic on if minerva story or miniature thumbnail are run out of samplesheet and into `params`
+
+## Minerva
+- ✅ ~~Ensure H&E images are rendered with a fixed legend~~
+- Allow for custom descriptions to be provided in the samplesheet and added into the Minerva story
+- Update to new version of Minerva with channel selection
+
+## Miniature
+
+- Split `make_miniature` into two processes, one to run the miniature script for multiplexed images and one (that can have lower resource requirements) that makes the brightfield thumbnail for H&E images with TiffSlide

assets/he_story.json

@@ -0,0 +1,21 @@
+{
+  "sample_info": {
+    "name": "",
+    "rotation": 0,
+    "text": "",
+    "pixels_per_micron": null
+  },
+  "defaults": [],
+  "groups": [
+    {
+      "label": "H&E",
+      "channels": [
+        { "color": "ffffff", "id": 0, "label": "H&E", "min": 0, "max": 1 }
+      ],
+      "render": [
+        { "color": "ffffff", "id": 0, "label": "H&E", "min": 0, "max": 1 }
+      ]
+    }
+  ],
+  "waypoints": []
+}

data/test_samplesheet.csv

@@ -0,0 +1,3 @@
+image,convert,he,minerva,miniature,id
+data/exemplar-001_small.tif,true,false,true,true,cycif
+data/CMU-1-Small-Region.svs,true,true,true,true,h_and_e

docker/Dockerfile

@@ -5,7 +5,7 @@
 FROM continuumio/miniconda3 AS build
 
 # File Author / Maintainer
-MAINTAINER Adam Taylor <[email protected]>
+LABEL maintainer="Adam Taylor <[email protected]>"
 
 # Install the package as normal:
 COPY environment.yml .
@@ -24,7 +24,6 @@ RUN conda-pack -n artist -o /tmp/env.tar && \
 # so now fix up paths:
 RUN /venv/bin/conda-unpack
 
-
 # The runtime-stage image; we can use Debian as the
 # base image since the Conda env also includes Python
 # for us.
@@ -48,15 +47,15 @@ RUN apt-get update && apt-get install --yes --no-install-recommends \
     libblosc1
 
 RUN source /venv/bin/activate && \
-    git clone https://github.com/adamjtaylor/miniature.git && \
-    git clone https://github.com/jmuhlich/auto-minerva.git && \
-    git clone https://github.com/labsyspharm/minerva-author.git && \
-    git clone https://github.com/ncihtan/image-header-validation && \
-    wget https://gist.githubusercontent.com/adamjtaylor/964e206bf1e6f302f6e512082e953193/raw/0acdea0736a027a260a0bb598f619552ff106758/index.html && \
+    git clone https://github.com/adamjtaylor/miniature.git -b v2 && \
+    git clone https://github.com/adamjtaylor/auto-minerva.git -b minerva1point5 && \
+    git clone https://github.com/labsyspharm/minerva-author.git -b v1.14.0 &&  \
     pip install git+https://github.com/labsyspharm/minerva-lib-python@master#egg=minerva-lib && \
     pip install openslide-python && \
     pip install opencv-python-headless \
-    pip install synapseclient
+    pip install synapseclient \
+    pip install mantel \
+    pip install ome_types
 
 ENV VIRTUAL_ENV=/venv
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"

docker/environment.yml

@@ -5,7 +5,7 @@ channels:
   - fastai
   - bioconda
 dependencies:
-  - numpy
+  - numpy=1.23.0
   - Pillow
   - flask
   - waitress
@@ -27,5 +27,8 @@ dependencies:
   - h5py
   - bioformats2raw=0.4.0
   - raw2ometiff
-  - ome-types
   - pip
+  - imagemagick
+  - tiffslide=2.1.2
+  - libvips
+  - estimagic