diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15e89de..f862389 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,54 +3,54 @@ name: ci on: push: branches: - - 'main' + - "main" paths: - - 'docker/**' - - '.github/workflows/ci.yml' + - "docker/**" + - ".github/workflows/ci.yml" env: REGISTRY: ghcr.io jobs: - docker: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - id-token: write - - defaults: - run: - working-directory: './docker' - - steps: - - name: Checkout GitHub Action - uses: actions/checkout@v3 - - - name: Login to GitHub Container Registry (GHCR) - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract Docker metadata - id: metadata - uses: docker/metadata-action@v4 - with: - images: ${{ env.REGISTRY }}/${{ github.repository }} - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=ref,event=branch - type=sha - latest - - - name: Build and push to GHCR - uses: docker/build-push-action@v4 - with: - context: ./docker - push: true - tags: ${{ steps.metadata.outputs.tags }} - labels: ${{ steps.metadata.outputs.labels }} + docker: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + defaults: + run: + working-directory: "./docker" + + steps: + - name: Checkout GitHub Action + uses: actions/checkout@v3 + + - name: Login to GitHub Container Registry (GHCR) + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: metadata + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ github.repository }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=ref,event=branch + type=sha + latest + + - name: Build and push to GHCR + uses: docker/build-push-action@v4 + with: + context: ./docker + push: true + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} diff --git a/.github/workflows/scan-images.yml b/.github/workflows/scan-images.yml index 2faa536..c0d73f7 100644 --- a/.github/workflows/scan-images.yml +++ b/.github/workflows/scan-images.yml @@ -1,22 +1,22 @@ name: Scan image on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: '32 12 * * 2' # every tuesday at 12:32 UTC time + push: + branches: ["main"] + pull_request: + branches: ["main"] + schedule: + - cron: "32 12 * * 2" # every tuesday at 12:32 UTC time env: - REGISTRY: ghcr.io - IMAGE_TAG: latest # scan just latest image + REGISTRY: ghcr.io + IMAGE_TAG: latest # scan just latest image jobs: trivy: - name: Run Trivy vulnerability scanner + name: Run Trivy vulnerability scanner permissions: - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results runs-on: ubuntu-latest steps: @@ -31,8 +31,8 @@ jobs: id: metadata uses: docker/metadata-action@v4 with: - images: ${{ env.REGISTRY }}/${{ github.repository }} - tags: ${{ env.IMAGE_TAG }} + images: ${{ env.REGISTRY }}/${{ github.repository }} + tags: ${{ env.IMAGE_TAG }} - name: Pull the image run: | @@ -43,7 +43,7 @@ jobs: with: image-ref: ${{ steps.metadata.outputs.tags }} ignore-unfixed: true - severity: 'CRITICAL,HIGH' + severity: "CRITICAL,HIGH" limit-severities-for-sarif: true format: sarif output: trivy-results.sarif diff --git a/.gitignore b/.gitignore index 554f083..23227d6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ work/ .nextflow.log* test-outputs/ flowchart* -outputs/* \ No newline at end of file +outputs/* +.DS_Store \ No newline at end of file diff --git a/README.md b/README.md index 50b82d0..5c2d0d1 100644 --- a/README.md +++ b/README.md @@ -3,50 +3,68 @@ A NextFlow pipeline to run image rendering process to generate resources for the [HTAN Portal](https://github.com/ncihtan/htan-portal). - Converts bioformats files into OME-TIFF -- Generates a `story.json` file using [Auto-Minerva](https://github.com/jmuhlich/auto-minerva) +- Sets thresholds for each channel and pepares 4-channel overlay groups using [Auto-Minerva](https://github.com/jmuhlich/auto-minerva) - Renders a Minerva story using [Minerva Author](https://github.com/labsyspharm/minerva-author) -- If the `--all` parameter is set, renders a thumbnail, an autominerva story and gets the metadata -- If the `--miniature` parameter is set, renders a thumbnail image using [Miniature](https://github.com/adamjtaylor/miniature) -- `--he` assumes the channel is a brighfield microscopy image of H&E stained tissue and uses a fixed, unscaled `story.json` and a custom color legend -- `--input` can be the path to an image (with `*` wildcards) or a csv manifest of cloud storage uris (one per line). +- Renders a thumbnail image using [Miniature](https://github.com/adamjtaylor/miniature) A Docker container ([ghcr.io/sage-bionetworks-workflows/nf-artist](https://github.com/sage-bionetworks-workflows/nf-artist/pkgs/container/nf-artist)) is used to ensure reproducibility. ## Example usage ``` -nextflow run ghcr.io/sage-bionetworks-workflows/nf-artist --input_path --outdir --all +nextflow run ghcr.io/sage-bionetworks-workflows/nf-artist --input --outdir +``` + + +## Output + +`nf-artist` outputs the following directory structure into the specified output directory (`outdir`): + +``` +├── outdir +│ ├── +│ │ ├── thumbnail.jpeg +│ │ ├── minerva +│ │ │ ├── index.html +│ │ │ ├── exhibit.json +│ │ │ ├── story.json +│ │ │ ├── Group-1 +│ │ │ │ ├── tile1.jpeg +│ │ │ │ ├── ... +│ │ │ ├── Group- +│ │ │ │ ├── tile1.jpeg +│ │ │ │ ├── ... +│ ├── < simpleName or id for n'th row of samplesheet> ``` ## Options -`--outdir` - output directory. Default: `.` -`--minerva`: Renders an [Auto-Minerva](https://github.com/jmuhlich/auto-minerva) story -`--miniature` - Renders a thumbnail image using [Miniature](https://github.com/adamjtaylor/miniature) -`--metadata` - Extract headers from the image and save as a json -`--all` - set `--minerva` `--miniature` and `--metadata` -`--he` - Use an unscaled scene for Minerva story and thumbnail generation. Suitable for H&E images -`--input_csv` - Path to a csv with a file path, uid, or synapseID per row -`--input_synid` - A synapse ID -`--input_path` - The path to a file. Can take wildcards -`--watch_path` - A path to watch for files that are created or modified -`--watch_csv` - A path to a csv to watch for if it is modified -`--echo` - Echo outputs -`--keepBg` - Keep the background in thumbnails -`--level` - the pyramid level used in thumbnauls, Default: `-1` (highest) -`--bioformats2ometiff` - Convert images to ome-tiff. Default: `true` -`--synapseconfig` - Path to a synapseConfig file. Required for Synapse authentication +#### Input/Output Options: + +* **input**: Path to a CSV sample sheet. This parameter is required. (Type: String) + +* **outdir**: Specifies the directory where the output data should be saved. Default is "outputs". (Type: String) + +#### Miniature Options: + +* **remove_bg**: Setting this to true will remove the non-tissue background. Default is true. (Type: Boolean) + +* **level**: Specifies the pyramid level used in thumbnails. Default is -1 (smallest). (Type: Integer) -## Example flow diagram: +* **dimred**: The dimensionality reduction method used. Default is "umap". Options include "umap", "tsne", and "pca". (Type: String) -![image](https://user-images.githubusercontent.com/14945787/133272620-18223615-ce22-41c3-807b-3f3007b8f080.png) +* **colormap**: Specifies the colormap used. Ensure the colormap is compatible with the number of `n_components` selected. Default is "UCIE". 3D colormap options: "UCIE", "LAB", "RGB". 2D colormap options: "BREMM", "SCHUMANN", "STEIGER", "TEULING2", "ZIEGLER", "CUBEDIAGONAL". (Type: String) -## Docker pointers +* **n_components**: Specifies the number of components. Default is 3. Options are 2 and 3. (Type: Integer) -### Test docker container +#### Samplesheet requirements -`docker run -ti ghcr.io/sage-bionetworks-workflows/nf-artist` +The samplesheet specified in the `input` parameter should be a CSV file with the following columns -## Build docker container +- `image`: [string] Path or URI to image to be processed +- `convert`: [boolean] Should the image be converted to a OME-TIFF +- `he`: [boolean] Is the image a H&E image +- `minerva`: [boolean] Should a Minerva story be generated +- `miniatuee`: [boolean] Should a Miniature thumbnail be generated +- `id`: *optional* [string] A custom identifier to replace image simpleName in output directory structure -`docker build -t ghcr.io/sage-bionetworks-workflows/nf-artist docker/` diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..f2bb1e0 --- /dev/null +++ b/TODO.md @@ -0,0 +1,14 @@ +# nf-artist +nf-artist is a Nextflow pipeline to generate interactive multiplexed image explorations and thumbnails. + +- Add CI testing in Github action using included test data in `data/` and samplesheet `data/test_samplesheet.csv` +- Move logic on if minerva story or miniature thumbnail are run out of samplesheet and into `params` + +## Minerva +- ✅ ~~Ensure H&E images are rendered with a fixed legend~~ +- Allow for custom descriptions to be provided in the samplesheet and added into the Minerva story +- Update to new version of Minerva with channel selection + +## Miniature + +- Split `make_miniature` into two processes, one to run the miniature script for multiplexed images and one (that can have lower resource requirements) that makes the brightfield thumbnail for H&E images with TiffSlide \ No newline at end of file diff --git a/assets/he_story.json b/assets/he_story.json new file mode 100644 index 0000000..175ccd6 --- /dev/null +++ b/assets/he_story.json @@ -0,0 +1,21 @@ +{ + "sample_info": { + "name": "", + "rotation": 0, + "text": "", + "pixels_per_micron": null + }, + "defaults": [], + "groups": [ + { + "label": "H&E", + "channels": [ + { "color": "ffffff", "id": 0, "label": "H&E", "min": 0, "max": 1 } + ], + "render": [ + { "color": "ffffff", "id": 0, "label": "H&E", "min": 0, "max": 1 } + ] + } + ], + "waypoints": [] +} diff --git a/data/CMU-1-Small-Region.svs b/data/CMU-1-Small-Region.svs new file mode 100644 index 0000000..1125dcd Binary files /dev/null and b/data/CMU-1-Small-Region.svs differ diff --git a/data/cycif_tonsil_small.tiff b/data/cycif_tonsil_small.tiff deleted file mode 100644 index b64d934..0000000 Binary files a/data/cycif_tonsil_small.tiff and /dev/null differ diff --git a/data/cycif_tonsil_very_small.ome.tiff b/data/cycif_tonsil_very_small.ome.tiff deleted file mode 100644 index 62db5a8..0000000 Binary files a/data/cycif_tonsil_very_small.ome.tiff and /dev/null differ diff --git a/data/exemplar-001_small.tif b/data/exemplar-001_small.tif new file mode 100644 index 0000000..4b5a24e Binary files /dev/null and b/data/exemplar-001_small.tif differ diff --git a/data/test.csv b/data/test.csv deleted file mode 100644 index 04161e3..0000000 --- a/data/test.csv +++ /dev/null @@ -1,2 +0,0 @@ -image,he -data/cycif_tonsil_very_small.ome.tiff, false \ No newline at end of file diff --git a/data/test_samplesheet.csv b/data/test_samplesheet.csv new file mode 100644 index 0000000..641e932 --- /dev/null +++ b/data/test_samplesheet.csv @@ -0,0 +1,3 @@ +image,convert,he,minerva,miniature,id +datas/exemplar-001_small.tif,true,false,true,true,cycif +data/CMU-1-Small-Region.svs,true,true,true,true,h_and_e \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 2bdd127..70da05d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,7 +24,6 @@ RUN conda-pack -n artist -o /tmp/env.tar && \ # so now fix up paths: RUN /venv/bin/conda-unpack - # The runtime-stage image; we can use Debian as the # base image since the Conda env also includes Python # for us. @@ -48,14 +47,9 @@ RUN apt-get update && apt-get install --yes --no-install-recommends \ libblosc1 RUN source /venv/bin/activate && \ - #git clone https://github.com/adamjtaylor/miniature.git && \ git clone https://github.com/adamjtaylor/miniature.git -b v2 && \ - #git clone https://github.com/jmuhlich/auto-minerva.git && \ git clone https://github.com/adamjtaylor/auto-minerva.git -b minerva1point5 && \ - git clone https://github.com/labsyspharm/minerva-author.git minerva-author-dev && \ - git clone https://github.com/labsyspharm/minerva-author.git -b v1.11.2 && \ - git clone https://github.com/ncihtan/image-header-validation && \ - wget https://gist.githubusercontent.com/adamjtaylor/964e206bf1e6f302f6e512082e953193/raw/0acdea0736a027a260a0bb598f619552ff106758/index.html && \ + git clone https://github.com/labsyspharm/minerva-author.git -b v1.14.0 && \ pip install git+https://github.com/labsyspharm/minerva-lib-python@master#egg=minerva-lib && \ pip install openslide-python && \ pip install opencv-python-headless \ diff --git a/docker/environment.yml b/docker/environment.yml index d0b38e7..df9f387 100644 --- a/docker/environment.yml +++ b/docker/environment.yml @@ -31,4 +31,4 @@ dependencies: - imagemagick - tiffslide=2.1.2 - libvips - - estimagic \ No newline at end of file + - estimagic diff --git a/main.nf b/main.nf index bd77000..c938f01 100644 --- a/main.nf +++ b/main.nf @@ -12,12 +12,6 @@ params.dimred = "umap" params.colormap = "UCIE" params.n_components = 3 -// made these into params so that they can be accessed anywhere in the pipeline -// can be added to profiles or other configuration later -params.heStory = 'https://gist.githubusercontent.com/adamjtaylor/3494d806563d71c34c3ab45d75794dde/raw/d72e922bc8be3298ebe8717ad2b95eef26e0837b/unscaled.story.json' -params.heScript = 'https://gist.githubusercontent.com/adamjtaylor/bbadf5aa4beef9aa1d1a50d76e2c5bec/raw/1f6e79ab94419e27988777343fa2c345a18c5b1b/fix_he_exhibit.py' -params.minerva_description_script = 'https://gist.githubusercontent.com/adamjtaylor/e51873a801fee39f1f1efa978e2b5e44/raw/c03d0e09ec58e4c391f5ce4ca4183abca790f2a2/inject_description.py' - include { ARTIST } from './workflows/artist.nf' workflow NF_ARTIST { diff --git a/modules/autominerva_story.nf b/modules/autominerva_story.nf index fab3f04..9d48048 100644 --- a/modules/autominerva_story.nf +++ b/modules/autominerva_story.nf @@ -1,23 +1,19 @@ process autominerva_story { + tag {"$meta.id"} + label "process_low" input: tuple val(meta), file(image) output: tuple val(meta), file(image), file('story.json') publishDir "$params.outdir/$workflow.runName", - pattern: 'story.json', - saveAs: {filename -> "${meta.id}/$workflow.runName/story.json"} + pattern: 'story.json', + saveAs: {filename -> "${meta.id}/$workflow.runName/story.json"} stub: """ touch story.json """ script: - if (meta.he) { - """ - wget -O story.json $params.heStory - """ - } else { - """ - python3 /auto-minerva/story.py $image > 'story.json' - """ - } + """ + python3 /auto-minerva/story.py $image > 'story.json' + """ } diff --git a/modules/bioformats2ometiff.nf b/modules/bioformats2ometiff.nf index e13b43d..2615907 100644 --- a/modules/bioformats2ometiff.nf +++ b/modules/bioformats2ometiff.nf @@ -1,4 +1,5 @@ process bioformats2ometiff { + tag {"$meta.id"} label "process_medium" input: tuple val(meta), file(image) diff --git a/modules/make_miniature.nf b/modules/make_miniature.nf index 8999e16..83e2927 100644 --- a/modules/make_miniature.nf +++ b/modules/make_miniature.nf @@ -1,5 +1,6 @@ process make_miniature { - label "process_medium" + tag {"$meta.id"} + label "process_high" input: tuple val(meta), file(image) output: diff --git a/modules/minerva.nf b/modules/minerva.nf deleted file mode 100644 index 6636fe5..0000000 --- a/modules/minerva.nf +++ /dev/null @@ -1,93 +0,0 @@ - -process STORY { - label "process_medium" - publishDir "$params.outdir/$workflow.runName" - mode: 'copy' - - echo params.echo - - when: - params.minerva == true || params.all == true - - input: - tuple val(synid), file(ome) - - output: - tuple val(synid), file('story.json'), file(ome) - - stub: - """ - touch story.json - """ - - script: - """ - python3 /auto-minerva/story.py $ome > 'story.json' - """ -} - - -process RENDER { - label "process_medium" - publishDir "$params.outdir/$workflow.runName", - mode: 'move' - - echo params.echo - - input: - tuple (val); synid, file(story), file(ome) - file synapseconfig from synapseconfig - - output: - file 'minerva' - - stub: - """ - mkdir minerva - touch minerva/index.html - touch minerva/exhibit.json - """ - - script: - """ - python3 /minerva-author/src/save_exhibit_pyramid.py $ome $story 'minerva' - cp /index.html minerva - wget -O inject_description.py $params.minerva_description_script - python3 inject_description.py minerva/exhibit.json --synid $synid --output minerva/exhibit.json --synapseconfig $synapseconfig - """ -} - -process RENDER_HE { - label "process_medium" - - publishDir "$params.outdir/$workflow.runName", - mode: 'move' - - echo params.echo - - input: - tuple val(synid), file(ome) - file synapseconfig from synapseconfig - - output: - file 'minerva' - - stub: - """ - mkdir minerva - touch minerva/index.html - touch minerva/exhibit.json - """ - - script: - """ - wget -O story.json $params.heStory - python3 /minerva-author/src/save_exhibit_pyramid.py $ome $story 'minerva' - cp /index.html minerva - wget -O fix_he_exhibit.py $params.heScript - python3 fix_he_exhibit.py minerva/exhibit.json - wget -O inject_description.py $params.minerva_description_script - python3 inject_description.py minerva/exhibit.json -synid$synid --synapseconfig $synapseconfig - """ -} - diff --git a/modules/render_pyramid.nf b/modules/render_pyramid.nf index 7df6c49..f0514f2 100644 --- a/modules/render_pyramid.nf +++ b/modules/render_pyramid.nf @@ -1,4 +1,6 @@ process render_pyramid { + tag {"$meta.id"} + label "process_medium" input: tuple val(meta), file(image), file (story) output: @@ -15,7 +17,6 @@ process render_pyramid { script: """ python3 /minerva-author/src/save_exhibit_pyramid.py $image $story 'minerva' - cp /index.html minerva """ } diff --git a/modules/synapse.nf b/modules/synapse.nf deleted file mode 100644 index ca65ef4..0000000 --- a/modules/synapse.nf +++ /dev/null @@ -1,17 +0,0 @@ -process SYNAPSE_GET { - label "process_low" - echo params.echo - input: - val synid - output: - tuple val(synid), file('*') - stub: - """ - touch "test.tif" - """ - script: - """ - echo "synapse -c $params.synapseconfig get $synid" - synapse -c $params.synapseconfig get $synid - """ -} \ No newline at end of file diff --git a/modules/synapse2.nf b/modules/synapse2.nf deleted file mode 100644 index 2971f5a..0000000 --- a/modules/synapse2.nf +++ /dev/null @@ -1,312 +0,0 @@ -// Create channel for synapse config -if (params.synapse_config) { - ch_synapse_config = file(params.synapse_config, checkIfExists: true) -} else { - exit 1, 'Please provide a Synapse config file for download authentication!' -} - -params.convert = true -params.outdir = 'test-outputs' -params.thumbnail_width = 512 -params.thumbnail_quality = 85 -params.remove_bg = true -params.level = -1 -params.dimred = 'umap' -heStory = 'https://gist.githubusercontent.com/adamjtaylor/3494d806563d71c34c3ab45d75794dde/raw/d72e922bc8be3298ebe8717ad2b95eef26e0837b/unscaled.story.json' - - - -workflow SYNAPSE { - - take: - ids - - - main: - - synapse_show ( - ids, - ch_synapse_config - ) - - // Get metadata into channels - synapse_show - .out - .metadata - .map { it -> synapseShowToMap(it) } - .set { ch_samples_meta } - - synapse_get ( - ch_samples_meta, - ch_synapse_config - ) - .set {synapse_out} - - emit: synapse_out - - -} - -workflow CONVERT { - take: images - main: - images - .filter { - it[0].ome == false - } - .set {bioformats} - - bioformats2ometiff( bioformats) - - images - .filter { - it[0].ome == true - } - .mix (bioformats2ometiff.out) - .set {converted} - - emit: converted -} - -workflow THUMBNAIL { - take: - converted - - main: - converted - .branch { - h_and_e: it[0].h_and_e == true - multiplex: it[0].h_and_e == false - } - .set { type } - - make_miniature ( type.multiplex ) - make_thumbnail ( type.h_and_e) - - make_miniature - .out - .mix ( make_thumbnail.out ) - .set { thumbnails } - - emit: thumbnails - -} - -workflow MINERVA { - take: - converted - - main: - - autominerva_story(converted) - render_pyramid(autominerva_story.out) -} - - -workflow { - SYNAPSE ( Channel.from('syn27056837','syn24829433') ) - CONVERT ( SYNAPSE.out) - THUMBNAIL ( CONVERT.out ) - MINERVA ( CONVERT.out ) -} - - -def synapseShowToMap(synapse_file) { - def meta = [:] - def category = '' - synapse_file.eachLine { line -> - def entries = [null, null] - if (!line.startsWith(' ') && !line.trim().isEmpty()) { - category = line.tokenize(':')[0] - } else { - entries = line.trim().tokenize('=') - } - meta["${category}|${entries[0]}"] = entries[1] - } - meta.id = meta['properties|id'] - meta.name = meta['properties|name'] - meta.md5 = meta['File|md5'] - meta.ome = meta['properties|name'] ==~ /.+\.ome\.tif{1,2}$/ - meta.h_and_e = meta['annotations|ImagingAssayType'] == "['H&E']" - return meta.findAll{ it.value != null } - } - -process synapse_show { - tag "$id" - label 'process_low' - - conda "bioconda::synapseclient=2.6.0" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/synapseclient:2.6.0--pyh5e36f6f_0' : - 'quay.io/biocontainers/synapseclient:2.6.0--pyh5e36f6f_0' }" - - input: - val id - path config - - output: - path "*.txt" , emit: metadata - - script: - def args = task.ext.args ?: '' - def args2 = task.ext.args2 ?: '' - """ - synapse \\ - -c $config \\ - show \\ - $args \\ - $id \\ - $args2 \\ - > ${id}.metadata.txt - rm $config - """ -} - -process synapse_get { - tag "$meta.id" - label 'process_low' - label 'error_retry' - - conda "bioconda::synapseclient=2.6.0" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/synapseclient:2.6.0--pyh5e36f6f_0' : - 'quay.io/biocontainers/synapseclient:2.6.0--pyh5e36f6f_0' }" - - input: - val meta - path config - - output: - tuple val(meta), path('*'), emit: image - - script: - def args = task.ext.args ?: '' - """ - synapse \\ - -c $config \\ - get \\ - $args \\ - $meta.id - shopt -s nullglob - for f in *\\ *; do mv "\${f}" "\${f// /_}"; done - """ -} - -process bioformats2ometiff { - label "process_medium" - input: - tuple val(meta), file(image) - output: - tuple val(meta), file("${image.simpleName}.ome.tiff") - stub: - """ - touch raw_dir - touch "${image.simpleName}.ome.tiff" - """ - script: - if ( meta.h_and_e){ - """ - bioformats2raw $image 'raw_dir' --series 0 - raw2ometiff 'raw_dir' "${image.simpleName}.ome.tiff" --rgb - """ - } else { - """ - bioformats2raw $image 'raw_dir' - raw2ometiff 'raw_dir' "${image.simpleName}.ome.tiff" - """ - } -} - -process make_miniature { - label "process_medium" - input: - tuple val(meta), file(image) - output: - tuple val(meta), file('data/miniature.jpg') - publishDir "$params.outdir/$workflow.runName", - saveAs: {filename -> "${meta.id}/$workflow.runName/thumbnail.jpg"} - stub: - """ - mkdir data - touch data/miniature.jpg - """ - script: - """ - mkdir data - python3 /miniature/docker/paint_miniature.py \ - $image 'miniature.jpg' \ - --remove_bg $params.remove_bg \ - --level $params.level \ - --dimred $params.dimred - """ -} - -process make_thumbnail { - label "process_medium" - errorStrategy 'ignore' - input: - tuple val(meta), file(image) - output: - tuple val(meta), file('miniature.jpg') - publishDir "$params.outdir/$workflow.runName", - saveAs: {filename -> "${meta.id}/$workflow.runName/thumbnail.jpg"} - stub: - """ - touch miniature.jpg - """ - script: - """ - convert $image \\ - -thumbnail '${params.thumbnail_width}x${params.thumbnail_width}>' \\ - -unsharp 0x.5 \\ - -scene 1 \\ - -quality $params.thumbnail_quality \\ - miniature.jpg - """ -} - - -process autominerva_story { - errorStrategy 'ignore' - input: - tuple val(meta), file(image) - output: - tuple val(meta), file(image), file('story.json') - publishDir "$params.outdir/$workflow.runName", - saveAs: {filename -> "${meta.id}/$workflow.runName/story.json"} - stub: - """ - touch story.json - """ - script: - if (meta.h_and_e) { - """ - wget -O story.json $heStory - """ - } else { - """ - python3 /auto-minerva/story.py $image > 'story.json' - """ - } -} - -process render_pyramid { - input: - tuple val(meta), file(image), file (story) - output: - tuple val(meta), path('minerva') - publishDir "$params.outdir/$workflow.runName", - saveAs: {filename -> "${meta.id}/$workflow.runName/minerva"} - stub: - """ - mkdir minerva - touch minerva/tile1.png - touch minerva/author.json - touch minerva/index.html - """ - script: - """ - python3 /minerva-author/src/save_exhibit_pyramid.py $image $story 'minerva' - cp /index.html minerva - """ -} \ No newline at end of file diff --git a/modules/thumbnails.nf b/modules/thumbnails.nf deleted file mode 100644 index f924710..0000000 --- a/modules/thumbnails.nf +++ /dev/null @@ -1,55 +0,0 @@ -process MINIATURE { - label "process_high" - - publishDir "$params.outdir/$workflow.runName", - saveAs: "thumbnail.png" - - echo params.echo - - input: - tuple val(synid), file(ome) - - output: - file 'data/miniature.png' - - stub: - """ - mkdir data - touch data/miniature.png - """ - - script: - """ - mkdir data - python3 /miniature/docker/paint_miniature.py $ome 'miniature.png' --remove_bg $remove_bg --level $params.level - """ -} - - -process RESIZE { - label "process_high" - - publishDir "$params.outdir/$workflow.runName", - saveAs: "thumbnail.png" - - echo params.echo - - input: - tuple val(synid), file(ome) - - output: - file 'data/miniature.png' - - stub: - """ - mkdir data - touch data/miniature.png - """ - - script: - """ - mkdir data - magick $ome -resize 512x512 data/miniature.png - """ -} - diff --git a/subworkflows/minerva.nf b/subworkflows/minerva.nf index 4c377ee..3abc19c 100644 --- a/subworkflows/minerva.nf +++ b/subworkflows/minerva.nf @@ -6,11 +6,23 @@ workflow MINERVA { converted main: + + // If H&E, used a fixed story from the assets dir converted .filter { - it[0].minerva == true - } - .set {for_minerva } - autominerva_story(for_minerva) - render_pyramid(autominerva_story.out) + it[0].minerva && it[0].he + } + .map { it -> [it[0], it[1], file( "$projectDir/assets/he_story.json", checkIfExists: true)] } + .set { he_story } + + // If not H&E, run auto-minerva + converted + .filter { + it[0].minerva && it[0].he == false + } | + autominerva_story | + // Mix with the `he_story` channel and render the pyramid + mix( he_story ) | + render_pyramid + } diff --git a/subworkflows/samplesheet_split.nf b/subworkflows/samplesheet_split.nf index a08ac29..c6076b8 100644 --- a/subworkflows/samplesheet_split.nf +++ b/subworkflows/samplesheet_split.nf @@ -9,12 +9,7 @@ workflow SAMPLESHEET_SPLIT { .map { row -> def meta = [:] - if (row.id ) { - meta.id = row.id - } else { - meta.id = file(row.image).simpleName - - } + meta.id = file(row.image).simpleName meta.ome = row.image ==~ /.+\.ome\.tif{1,2}$/ meta.convert = row.convert.toBoolean() meta.he = row.he.toBoolean() diff --git a/test.csv b/test.csv deleted file mode 100644 index 78da384..0000000 --- a/test.csv +++ /dev/null @@ -1,2 +0,0 @@ -syn1234 -syn12345 \ No newline at end of file diff --git a/test.nf b/test.nf deleted file mode 100644 index 4f68c36..0000000 --- a/test.nf +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env nextflow - -// Enable dsl2 -nextflow.enable.dsl=2 - -params.input_csv = './data/test.csv' -params.input_path = false - -process SYNAPSE_GET { - label "process_low" - echo params.echo - input: - val synid - output: - tuple val(synid), file('*') - stub: - """ - touch "test.tif" - """ - script: - """ - synapse -c ~/.synapseConfig get $synid - """ -} - -Channel - .fromPath(params.input_csv) - .splitText() - .set { input_csv } - -//input_csv.view() - -if (params.input_path != false) { - Channel - .fromPath(params.input_path) - .set { input_path } -} else { - Channel.empty().set{input_path} -} - -//input_path.view() - -Channel - .empty() - .mix(input_path) - .mix(input_csv) - .branch { - syn: it =~ /^syn\d+$/ - other: true - } - .set { inputs } - -inputs.other - .map { it -> file(it) } - .map { it -> tuple(it.simpleName, it)} - .set {files} - -// files.view() - -workflow { - SYNAPSE_GET(inputs.syn) - SYNAPSE_GET.out.view() - files - .mix( - SYNAPSE_GET.out - ) - .view() -}