diff --git a/.editorconfig b/.editorconfig index b6b31907..9b990088 100644 --- a/.editorconfig +++ b/.editorconfig @@ -22,3 +22,11 @@ indent_size = unset [/assets/email*] indent_size = unset + +# ignore Readme +[README.md] +indent_style = unset + +# ignore python +[*.{py}] +indent_style = unset diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 500aea62..a8583ba0 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -27,6 +27,9 @@ If you're not used to this workflow with git, you can start with some [docs from ## Tests +You can optionally test your changes by running the pipeline locally. Then it is recommended to use the `debug` profile to +receive warnings about process selectors and other debug info. Example: `nextflow run . -profile debug,test,docker --outdir `. + When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests. Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6278e905..8adc195d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -19,6 +19,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/cuta - [ ] If necessary, also make a PR on the nf-core/cutandrun _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). +- [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. - [ ] `CHANGELOG.md` is updated. diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index 87c68811..e07796cc 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -28,7 +28,7 @@ jobs: } profiles: test_full - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: Tower debug log file path: | diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml index a2b2f308..fc49e48f 100644 --- a/.github/workflows/awstest.yml +++ b/.github/workflows/awstest.yml @@ -25,7 +25,7 @@ jobs: } profiles: test - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: Tower debug log file path: | diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index a0faf3c0..42a84a90 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -19,7 +19,7 @@ jobs: # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets - name: Post PR comment if: failure() - uses: mshick/add-pr-comment@v1 + uses: mshick/add-pr-comment@v2 with: message: | ## This PR is against the `master` branch :x: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7765aa0..f58a6c7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,7 +110,7 @@ jobs: CAPSULE_LOG: none steps: - name: Check out pipeline code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v1 diff --git a/.github/workflows/clean-up.yml b/.github/workflows/clean-up.yml index 694e90ec..e37cfda5 100644 --- a/.github/workflows/clean-up.yml +++ b/.github/workflows/clean-up.yml @@ -10,7 +10,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v7 + - uses: actions/stale@v9 with: stale-issue-message: "This issue has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days." stale-pr-message: "This PR has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment if it is still useful." diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml new file mode 100644 index 00000000..8a330045 --- /dev/null +++ b/.github/workflows/download_pipeline.yml @@ -0,0 +1,67 @@ +name: Test successful pipeline download with 'nf-core download' + +# Run the workflow when: +# - dispatched manually +# - when a PR is opened or reopened to master branch +# - the head branch of the pull request is updated, i.e. if fixes for a release are pushed last minute to dev. +on: + workflow_dispatch: + pull_request: + types: + - opened + branches: + - master + pull_request_target: + branches: + - master + +env: + NXF_ANSI_LOG: false + +jobs: + download: + runs-on: ubuntu-latest + steps: + - name: Install Nextflow + uses: nf-core/setup-nextflow@v1 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + architecture: "x64" + - uses: eWaterCycle/setup-singularity@v7 + with: + singularity-version: 3.8.3 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install git+https://github.com/nf-core/tools.git@dev + + - name: Get the repository name and current branch set as environment variable + run: | + echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> ${GITHUB_ENV} + echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> ${GITHUB_ENV} + echo "REPO_BRANCH=${GITHUB_REF#refs/heads/}" >> ${GITHUB_ENV} + + - name: Download the pipeline + env: + NXF_SINGULARITY_CACHEDIR: ./ + run: | + nf-core download ${{ env.REPO_LOWERCASE }} \ + --revision ${{ env.REPO_BRANCH }} \ + --outdir ./${{ env.REPOTITLE_LOWERCASE }} \ + --compress "none" \ + --container-system 'singularity' \ + --container-library "quay.io" -l "docker.io" -l "ghcr.io" \ + --container-cache-utilisation 'amend' \ + --download-configuration + + - name: Inspect download + run: tree ./${{ env.REPOTITLE_LOWERCASE }} + + - name: Run the downloaded pipeline + env: + NXF_SINGULARITY_CACHEDIR: ./ + NXF_SINGULARITY_HOME_MOUNT: true + run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index d16134b2..ceb666d3 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -4,7 +4,7 @@ on: types: [created] jobs: - deploy: + fix-linting: # Only run if comment is on a PR with the main repo, and if it contains the magic keywords if: > contains(github.event.comment.html_url, '/pull/') && @@ -13,10 +13,17 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@v3 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} + # indication that the linting is being fixed + - name: React on comment + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: eyes + # Action runs on the issue comment, so we don't get the PR by default # Use the gh cli to check out the PR - name: Checkout Pull Request @@ -24,32 +31,59 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - - uses: actions/setup-node@v3 + # Install and run pre-commit + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + with: + python-version: 3.11 - - name: Install Prettier - run: npm install -g prettier @prettier/plugin-php + - name: Install pre-commit + run: pip install pre-commit - # Check that we actually need to fix something - - name: Run 'prettier --check' - id: prettier_status - run: | - if prettier --check ${GITHUB_WORKSPACE}; then - echo "result=pass" >> $GITHUB_OUTPUT - else - echo "result=fail" >> $GITHUB_OUTPUT - fi + - name: Run pre-commit + id: pre-commit + run: pre-commit run --all-files + continue-on-error: true - - name: Run 'prettier --write' - if: steps.prettier_status.outputs.result == 'fail' - run: prettier --write ${GITHUB_WORKSPACE} + # indication that the linting has finished + - name: react if linting finished succesfully + if: steps.pre-commit.outcome == 'success' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: "+1" - name: Commit & push changes - if: steps.prettier_status.outputs.result == 'fail' + id: commit-and-push + if: steps.pre-commit.outcome == 'failure' run: | git config user.email "core@nf-co.re" git config user.name "nf-core-bot" git config push.default upstream git add . git status - git commit -m "[automated] Fix linting with Prettier" + git commit -m "[automated] Fix code linting" git push + + - name: react if linting errors were fixed + id: react-if-fixed + if: steps.commit-and-push.outcome == 'success' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: hooray + + - name: react if linting errors were not fixed + if: steps.commit-and-push.outcome == 'failure' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: confused + + - name: react if linting errors were not fixed + if: steps.commit-and-push.outcome == 'failure' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + issue-number: ${{ github.event.issue.number }} + body: | + @${{ github.actor }} I tried to fix the linting errors, but it didn't work. Please fix them manually. + See [CI log](https://github.com/nf-core/cutandrun/actions/runs/${{ github.run_id }}) for more details. diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index b8bdd214..81cd098e 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -11,72 +11,33 @@ on: types: [published] jobs: - EditorConfig: + pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 - - - name: Install editorconfig-checker - run: npm install -g editorconfig-checker - - - name: Run ECLint check - run: editorconfig-checker -exclude README.md $(find .* -type f | grep -v '.git\|.py\|.md\|json\|yml\|yaml\|html\|css\|work\|.nextflow\|build\|nf_core.egg-info\|log.txt\|Makefile') - - Prettier: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-node@v3 - - - name: Install Prettier - run: npm install -g prettier - - - name: Run Prettier --check - run: prettier --check ${GITHUB_WORKSPACE} - - PythonBlack: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Check code lints with Black - uses: psf/black@stable - - # If the above check failed, post a comment on the PR explaining the failure - - name: Post PR comment - if: failure() - uses: mshick/add-pr-comment@v1 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 with: - message: | - ## Python linting (`black`) is failing - - To keep the code consistent with lots of contributors, we run automated code consistency checks. - To fix this CI test, please run: - - * Install [`black`](https://black.readthedocs.io/en/stable/): `pip install black` - * Fix formatting errors in your pipeline: `black .` - - Once you push these changes the test should pass, and you can hide this comment :+1: + python-version: 3.11 + cache: "pip" - We highly recommend setting up Black in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + - name: Install pre-commit + run: pip install pre-commit - Thanks again for your contribution! - repo-token: ${{ secrets.GITHUB_TOKEN }} - allow-repeats: false + - name: Run pre-commit + run: pre-commit run --all-files nf-core: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v1 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" architecture: "x64" @@ -99,7 +60,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: linting-logs path: | diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml index 0bbcd30f..147bcd10 100644 --- a/.github/workflows/linting_comment.yml +++ b/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v3 with: workflow: linting.yml workflow_conclusion: completed diff --git a/.github/workflows/release-announcments.yml b/.github/workflows/release-announcements.yml similarity index 96% rename from .github/workflows/release-announcments.yml rename to .github/workflows/release-announcements.yml index 6ad33927..21ac3f06 100644 --- a/.github/workflows/release-announcments.yml +++ b/.github/workflows/release-announcements.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.10" - name: Install dependencies @@ -56,7 +56,7 @@ jobs: bsky-post: runs-on: ubuntu-latest steps: - - uses: zentered/bluesky-post-action@v0.0.2 + - uses: zentered/bluesky-post-action@v0.1.0 with: post: | Pipeline release! ${{ github.repository }} v${{ github.event.release.tag_name }} - ${{ github.event.release.name }}! diff --git a/.gitpod.yml b/.gitpod.yml index 25488dcc..363d5b1d 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -4,6 +4,9 @@ tasks: command: | pre-commit install --install-hooks nextflow self-update + - name: unset JAVA_TOOL_OPTIONS + command: | + unset JAVA_TOOL_OPTIONS vscode: extensions: # based on nf-core.nf-core-extensionpack diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0c31cdb9..af57081f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,10 @@ repos: - repo: https://github.com/pre-commit/mirrors-prettier - rev: "v2.7.1" + rev: "v3.1.0" hooks: - id: prettier + - repo: https://github.com/editorconfig-checker/editorconfig-checker.python + rev: "2.7.3" + hooks: + - id: editorconfig-checker + alias: ec diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a3ce089..503590cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.2.2] - 2024-02-01 + +### Enhancements + +- Updated pipeline template to nf-core/tools `2.12`. +- Added option to dump calculated scale factor values from the pipeline using `--dump_scale_factors` +- Added options to group IGV tracks by samplesheet group or by file type using `--igv_sort_by_groups` + +### Fixes + +- Fixed error that caused mismapping of IgG controls to their targets in certain samplesheet configurations. + ## [3.2.1] - 2023-10-22 ### Enhancements diff --git a/LICENSE b/LICENSE index 40528b68..27af3e07 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) Chris Cheshire and Charlotte West +Copyright (c) Chris Cheshire, Charlotte West and Tamara Hodgetts Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 54daf337..caa78e16 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ -# ![nf-core/cutandrun](docs/images/nf-core-cutandrun_logo_light.png#gh-light-mode-only) ![nf-core/cutandrun](docs/images/nf-core-cutandrun_logo_dark.png#gh-dark-mode-only) - +

+ + + nf-core/cutandrun + +

[![GitHub Actions CI Status](https://github.com/nf-core/cutandrun/workflows/nf-core%20CI/badge.svg)](https://github.com/nf-core/cutandrun/actions?query=workflow%3A%22nf-core+CI%22) [![GitHub Actions Linting Status](https://github.com/nf-core/cutandrun/workflows/nf-core%20linting/badge.svg)](https://github.com/nf-core/cutandrun/actions?query=workflow%3A%22nf-core+linting%22) [![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?logo=Amazon%20AWS)](https://nf-co.re/cutandrun/results) @@ -57,10 +61,8 @@ The pipeline has been developed with continuous integration (CI) and test driven ## Usage -> **Note** -> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how -> to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) -> with `-profile test` before running the workflow on actual data. +> [!NOTE] +> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data. First, prepare a samplesheet with your input data that looks as follows: @@ -85,9 +87,8 @@ nextflow run nf-core/cutandrun \ --genome GRCh38 \ --outdir -> **Warning:** -> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those -> provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; +> [!WARNING] +> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; > see [docs](https://nf-co.re/usage/configuration#custom-configuration-files). - Typical command for CUT&Run/CUT&Tag/TIPseq analysis: diff --git a/assets/email_template.html b/assets/email_template.html index e5f5f518..a791c6e3 100644 --- a/assets/email_template.html +++ b/assets/email_template.html @@ -12,7 +12,7 @@ -

nf-core/cutandrun v${version}

+

nf-core/cutandrun ${version}

Run Name: $runName

<% if (!success){ @@ -50,4 +50,4 @@

Pipeline Configuration:

- \ No newline at end of file + diff --git a/assets/email_template.txt b/assets/email_template.txt index 3d4bd78f..07c4f950 100644 --- a/assets/email_template.txt +++ b/assets/email_template.txt @@ -4,7 +4,7 @@ |\\ | |__ __ / ` / \\ |__) |__ } { | \\| | \\__, \\__/ | \\ |___ \\`-._,-`-, `._,._,' - nf-core/cutandrun v${version} + nf-core/cutandrun ${version} ---------------------------------------------------- Run Name: $runName diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index c77640db..a5b42b58 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -1,8 +1,8 @@ report_comment: > - This report has been generated by the nf-core/cutandrun + This report has been generated by the nf-core/cutandrun analysis pipeline. For information about how to interpret these results, please see the - documentation. + documentation. export_plots: true diff --git a/assets/nf-core-cutandrun_logo_light.png b/assets/nf-core-cutandrun_logo_light.png index b1ce4609..2dfd4370 100644 Binary files a/assets/nf-core-cutandrun_logo_light.png and b/assets/nf-core-cutandrun_logo_light.png differ diff --git a/assets/slackreport.json b/assets/slackreport.json index 17fdaab4..a52b5eb5 100644 --- a/assets/slackreport.json +++ b/assets/slackreport.json @@ -3,7 +3,7 @@ { "fallback": "Plain-text summary of the attachment.", "color": "<% if (success) { %>good<% } else { %>danger<%} %>", - "author_name": "nf-core/cutandrun v${version} - ${runName}", + "author_name": "nf-core/cutandrun ${version} - ${runName}", "author_icon": "https://www.nextflow.io/docs/latest/_static/favicon.ico", "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors<% } %>", "fields": [ diff --git a/bin/check_samplesheet.py b/bin/check_samplesheet.py index 628a54b8..94be56f0 100755 --- a/bin/check_samplesheet.py +++ b/bin/check_samplesheet.py @@ -244,6 +244,34 @@ def check_samplesheet(file_in, file_out, use_control): "WARNING: Parameter --use_control was set to false, but an control group was found in " + str(file_in) + "." ) + # Calculate the exact control/replicate id combo + if use_control == "true": + for group, reps in sorted(sample_run_dict.items()): + # Calculate the ctrl group + ctrl_group = None + is_ctrl = False + for replicate, info in sorted(reps.items()): + ctrl_group = info[0][2] + if ctrl_group == "": + is_ctrl = True + break + + # Continue if ctrl + if is_ctrl: + continue + + # Get num reps + num_reps = len(reps) + num_ctrl_reps = len(sample_run_dict[ctrl_group]) + + # Assign actual ctrl rep id + for rep, info in sorted(reps.items()): + if num_reps == num_ctrl_reps: + ctrl_group_new = ctrl_group + "_" + str(rep) + else: + ctrl_group_new = ctrl_group + "_1" + info[0][2] = ctrl_group_new + ## Write validated samplesheet with appropriate columns if len(sample_run_dict) > 0: out_dir = os.path.dirname(file_out) @@ -267,7 +295,8 @@ def check_samplesheet(file_in, file_out, use_control): check_group = sample_run_dict[sample][replicate][0][2] for tech_rep in sample_run_dict[sample][replicate]: if tech_rep[2] != check_group: - print_error("Control group must match within technical replicates", tech_rep[2]) + tech_rep[2] = check_group + # print_error("Control group must match within technical replicates", tech_rep[2]) ## Write to file for idx, sample_info in enumerate(sample_run_dict[sample][replicate]): diff --git a/bin/igv_files_to_session.py b/bin/igv_files_to_session.py index 222b22cb..098ac61f 100755 --- a/bin/igv_files_to_session.py +++ b/bin/igv_files_to_session.py @@ -99,6 +99,15 @@ def igv_files_to_session(XMLOut, ListFile, Genome, GtfBed, PathPrefix=""): break fout.close() + ## Construct groups + groups = {} + group_num = 1 + for ifile, colour in fileList: + group = os.path.basename(ifile).split("_R")[0] + if group not in groups: + groups[group] = group_num + group_num = group_num + 1 + ## ADD RESOURCES SECTION XMLStr = '\n' XMLStr += '\n' % (Genome) @@ -159,8 +168,8 @@ def igv_files_to_session(XMLOut, ListFile, Genome, GtfBed, PathPrefix=""): ) elif extension in [".bw", ".bigwig", ".tdf", ".bedGraph", ".bedgraph"]: XMLStr += ( - '\t\t w << sendmail_html } [ 'sendmail', '-t' ].execute() << sendmail_html log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (sendmail)-" } catch (all) { // Catch failures and try with plaintext def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] - if ( mqc_report.size() <= max_multiqc_email_size.toBytes() ) { + if ( mqc_report != null && mqc_report.size() <= max_multiqc_email_size.toBytes() ) { mail_cmd += [ '-A', mqc_report ] } mail_cmd.execute() << email_html @@ -155,14 +158,16 @@ class NfcoreTemplate { } // Write summary e-mail HTML to a file - def output_d = new File("${params.outdir}/pipeline_info/") - if (!output_d.exists()) { - output_d.mkdirs() - } - def output_hf = new File(output_d, "pipeline_report.html") + def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") output_hf.withWriter { w -> w << email_html } - def output_tf = new File(output_d, "pipeline_report.txt") + FilesEx.copyTo(output_hf.toPath(), "${params.outdir}/pipeline_info/pipeline_report.html"); + output_hf.delete() + + // Write summary e-mail TXT to a file + def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") output_tf.withWriter { w -> w << email_txt } + FilesEx.copyTo(output_tf.toPath(), "${params.outdir}/pipeline_info/pipeline_report.txt"); + output_tf.delete() } // @@ -227,15 +232,14 @@ class NfcoreTemplate { // Dump pipeline parameters in a json file // public static void dump_parameters(workflow, params) { - def output_d = new File("${params.outdir}/pipeline_info/") - if (!output_d.exists()) { - output_d.mkdirs() - } - def timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') - def output_pf = new File(output_d, "params_${timestamp}.json") + def filename = "params_${timestamp}.json" + def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") def jsonStr = JsonOutput.toJson(params) - output_pf.text = JsonOutput.prettyPrint(jsonStr) + temp_pf.text = JsonOutput.prettyPrint(jsonStr) + + FilesEx.copyTo(temp_pf.toPath(), "${params.outdir}/pipeline_info/params_${timestamp}.json") + temp_pf.delete() } // diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy index 12df71e9..023723ab 100755 --- a/lib/WorkflowMain.groovy +++ b/lib/WorkflowMain.groovy @@ -22,7 +22,7 @@ class WorkflowMain { // // Validate parameters and print summary to screen // - public static void initialise(workflow, params, log) { + public static void initialise(workflow, params, log, args) { // Print workflow version and exit on --version if (params.version) { @@ -33,6 +33,8 @@ class WorkflowMain { // Check that a -profile or Nextflow config has been provided to run the pipeline NfcoreTemplate.checkConfigProvided(workflow, log) + // Check that the profile doesn't contain spaces and doesn't end with a trailing comma + checkProfile(workflow.profile, args, log) // Check that conda channels are set-up correctly if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { @@ -72,4 +74,16 @@ class WorkflowMain { } return val } + + // + // Exit pipeline if --profile contains spaces + // + private static void checkProfile(profile, args, log) { + if (profile.endsWith(',')) { + Nextflow.error "Profile cannot end with a trailing comma. Please remove the comma from the end of the profile string.\nHint: A common mistake is to provide multiple values to `-profile` separated by spaces. Please use commas to separate profiles instead,e.g., `-profile docker,test`." + } + if (args[0]) { + log.warn "nf-core pipelines do not accept positional arguments. The positional argument `${args[0]}` has been detected.\n Hint: A common mistake is to provide multiple values to `-profile` separated by spaces. Please use commas to separate profiles instead,e.g., `-profile docker,test`." + } + } } diff --git a/lib/nfcore_external_java_deps.jar b/lib/nfcore_external_java_deps.jar deleted file mode 100644 index 805c8bb5..00000000 Binary files a/lib/nfcore_external_java_deps.jar and /dev/null differ diff --git a/main.nf b/main.nf index aa04c7f1..f3b5b5eb 100644 --- a/main.nf +++ b/main.nf @@ -56,7 +56,7 @@ if (params.validate_params) { validateParameters() } -WorkflowMain.initialise(workflow, params, log) +WorkflowMain.initialise(workflow, params, log, args) /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/modules/local/custom_dumpsoftwareversions.nf b/modules/local/custom_dumpsoftwareversions.nf index eeb3d770..0fe4d309 100644 --- a/modules/local/custom_dumpsoftwareversions.nf +++ b/modules/local/custom_dumpsoftwareversions.nf @@ -2,10 +2,10 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda "bioconda::multiqc=1.15" + conda "bioconda::multiqc=1.19" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.15--pyhdfd78af_0' : - 'biocontainers/multiqc:1.15--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.19--pyhdfd78af_0' : + 'biocontainers/multiqc:1.19--pyhdfd78af_0' }" input: path versions diff --git a/modules/local/multiqc.nf b/modules/local/multiqc.nf index 172fc6e8..11e6760d 100644 --- a/modules/local/multiqc.nf +++ b/modules/local/multiqc.nf @@ -1,10 +1,10 @@ process MULTIQC { label 'process_single' - conda "bioconda::multiqc=1.17" + conda "bioconda::multiqc=1.19" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.17--pyhdfd78af_0' : - 'biocontainers/multiqc:1.17--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.19--pyhdfd78af_0' : + 'biocontainers/multiqc:1.19--pyhdfd78af_0' }" input: path multiqc_config diff --git a/modules/local/peak_counts.nf b/modules/local/peak_counts.nf index 8d89e4ea..59a14758 100644 --- a/modules/local/peak_counts.nf +++ b/modules/local/peak_counts.nf @@ -2,10 +2,10 @@ process PEAK_COUNTS { tag "$meta.id" label 'process_low' - conda "bioconda::bedtools=2.31.0" + conda "bioconda::bedtools=2.31.1" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/bedtools:2.31.0--hf5e1c6e_2' : - 'biocontainers/bedtools:2.31.0--hf5e1c6e_2' }" + 'https://depot.galaxyproject.org/singularity/bedtools:2.31.1--hf5e1c6e_0' : + 'biocontainers/bedtools:2.31.1--hf5e1c6e_0' }" input: tuple val(meta), path(bed) diff --git a/modules/local/peak_frip.nf b/modules/local/peak_frip.nf index b52027ca..64ebc387 100644 --- a/modules/local/peak_frip.nf +++ b/modules/local/peak_frip.nf @@ -2,10 +2,10 @@ process PEAK_FRIP { tag "$meta.id" label 'process_medium' - conda "bioconda::bedtools=2.31.0" + conda "bioconda::bedtools=2.31.1" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/bedtools:2.31.0--hf5e1c6e_2' : - 'biocontainers/bedtools:2.31.0--hf5e1c6e_2' }" + 'https://depot.galaxyproject.org/singularity/bedtools:2.31.1--hf5e1c6e_0' : + 'biocontainers/bedtools:2.31.1--hf5e1c6e_0' }" input: tuple val(meta), path(peaks_bed), path(fragments_bed), path(flagstat) diff --git a/modules/local/python/igv_session.nf b/modules/local/python/igv_session.nf index 9bdf7dcd..2d3f52ec 100644 --- a/modules/local/python/igv_session.nf +++ b/modules/local/python/igv_session.nf @@ -15,6 +15,7 @@ process IGV_SESSION { path beds path secondary_beds path bigwig + val sort_by_groups output: path('*.{txt,xml,bed,bigWig,fa,fai,fna,gtf,gff,narrowPeak,broadPeak,gz,tbi,bedGraph}', includeInputs:true) @@ -27,10 +28,17 @@ process IGV_SESSION { output = '' colours = [:] colour_pos = 0 + file_list = [] + + if(sort_by_groups) { + file_list = beds.collect{it.toString()}.sort() + file_list += secondary_beds.collect{it.toString()}.sort() + file_list += bigwig.collect{it.toString()}.sort() + } + else { + file_list = (bigwig + secondary_beds + beds).collect{ it.toString() }.sort() + } - file_list = beds.collect{it.toString()}.sort() - file_list += secondary_beds.collect{it.toString()}.sort() - file_list += bigwig.collect{it.toString()}.sort() for(file in file_list){ file_split = file.split('_R') group = file_split[0] diff --git a/modules/local/samtools_custom_view.nf b/modules/local/samtools_custom_view.nf index a56581d0..e476e6b9 100644 --- a/modules/local/samtools_custom_view.nf +++ b/modules/local/samtools_custom_view.nf @@ -2,10 +2,10 @@ process SAMTOOLS_CUSTOMVIEW { tag "$meta.id" label 'process_low' - conda "bioconda::samtools=1.17" + conda "bioconda::samtools=1.19.2" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : - 'biocontainers/samtools:1.17--h00cdaf9_0' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.19.2--h50ea8bc_0' : + 'biocontainers/samtools:1.19.2--h50ea8bc_0' }" input: tuple val(meta), path(bam), path(bai) diff --git a/nextflow.config b/nextflow.config index ba570c9f..7dbf472b 100644 --- a/nextflow.config +++ b/nextflow.config @@ -79,13 +79,13 @@ params { seacr_peak_threshold = 0.05 macs2_pvalue = null macs2_qvalue = 0.01 - macs_gsize = 2.7e9 + macs_gsize = 2700000000.0 macs2_narrow_peak = true macs2_broad_cutoff = 0.1 // Consensus Peaks consensus_peak_mode = 'group' - replicate_threshold = 1 + replicate_threshold = 1.0 // Reporting and Visualisation skip_reporting = false @@ -95,9 +95,11 @@ params { skip_peak_qc = false skip_preseq = false skip_multiqc = false + dump_scale_factors = false igv_show_gene_names = true - min_frip_overlap = 0.20 - min_peak_overlap = 0.20 + igv_sort_by_groups = true + min_frip_overlap = 0.2 + min_peak_overlap = 0.2 // Deeptools options dt_heatmap_gene_bodylen = 5000 @@ -127,7 +129,6 @@ params { version = false igenomes_base = "s3://ngi-igenomes/igenomes/" igenomes_ignore = false - debug = false singularity_pull_docker_container = false // Config options @@ -146,7 +147,7 @@ params { // Schema validation default options validationFailUnrecognisedParams = false validationLenientMode = false - validationSchemaIgnoreParams = 'igenomes_base,debug,genomes,callers,dedup_control_only,fragment_size,run_igv,run_multiqc,run_reporting,run_consensus_all,run_peak_calling,run_remove_dups,run_remove_linear_dups,run_mark_dups,run_read_filter,run_alignment,run_trim_galore_fastqc,run_cat_fastq,run_input_check,run_genome_prep,run_peak_qc,run_deeptools_qc,run_deeptools_heatmaps,run_preseq' + validationSchemaIgnoreParams = 'igenomes_base,genomes,callers,dedup_control_only,fragment_size,run_igv,run_multiqc,run_reporting,run_consensus_all,run_peak_calling,run_remove_dups,run_remove_linear_dups,run_mark_dups,run_read_filter,run_alignment,run_trim_galore_fastqc,run_cat_fastq,run_input_check,run_genome_prep,run_peak_qc,run_deeptools_qc,run_deeptools_heatmaps,run_preseq' validationShowHiddenParams = false validate_params = true } @@ -176,6 +177,7 @@ profiles { dumpHashes = true process.beforeScript = 'echo $HOSTNAME' cleanup = false + nextflow.enable.configProcessNamesValidation = true } conda { conda.enabled = true @@ -184,6 +186,7 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false + channels = ['conda-forge', 'bioconda', 'defaults'] apptainer.enabled = false } mamba { @@ -198,16 +201,16 @@ profiles { } docker { docker.enabled = true - docker.userEmulation = true conda.enabled = false singularity.enabled = false podman.enabled = false shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + docker.runOptions = '-u $(id -u):$(id -g)' } arm { - docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } singularity { singularity.enabled = true @@ -286,7 +289,7 @@ singularity.registry = 'quay.io' // Nextflow plugins plugins { - id 'nf-validation' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-validation@1.1.3' // Validation of pipeline parameters and creation of an input channel from a sample sheet } // Load igenomes.config if required @@ -296,10 +299,6 @@ if (!params.igenomes_ignore) { params.genomes = [:] } -if(!params.debug) { - nextflow.enable.configProcessNamesValidation = false -} - // Export these variables to prevent local Python/R libraries from conflicting with those in the container // The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container. // See https://apeltzer.github.io/post/03-julia-lang-nextflow/ for details on that. Once we have a common agreement on where to keep Julia packages, this is adjustable. @@ -334,12 +333,12 @@ dag { manifest { name = 'nf-core/cutandrun' - author = """Chris Cheshire and Charlotte West""" + author = """Chris Cheshire, Charlotte West and Tamara Hodgetts""" homePage = 'https://github.com/nf-core/cutandrun' description = """Analysis pipeline for CUT&RUN and CUT&TAG experiments that includes sequencing QC, spike-in normalisation, IgG control normalisation, peak calling and downstream peak analysis.""" mainScript = 'main.nf' nextflowVersion = '!>=23.04.0' - version = '3.2.1' + version = '3.2.2' doi = 'https://doi.org/10.5281/zenodo.5653535' } diff --git a/nextflow_schema.json b/nextflow_schema.json index 06a70218..026eab52 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -26,7 +26,8 @@ "type": "string", "format": "directory-path", "description": "The output directory where the results will be saved. You have to use absolute paths to store on Cloud infrastructure.", - "fa_icon": "fas fa-folder-open" + "fa_icon": "fas fa-folder-open", + "default": "./results" }, "multiqc_title": { "type": "string", @@ -69,6 +70,10 @@ "fa_icon": "fas fa-envelope", "help_text": "Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits. If set in your user config file (`~/.nextflow/config`) then you don't need to specify this on the command line for every run.", "pattern": "^([a-zA-Z0-9_\\-\\.]+)@([a-zA-Z0-9_\\-\\.]+)\\.([a-zA-Z]{2,5})$" + }, + "dump_scale_factors": { + "type": "boolean", + "description": "Output calculated scale factors from pipeline" } } }, @@ -84,10 +89,26 @@ "fa_icon": "fas fa-book", "help_text": "If using a reference genome configured in the pipeline using iGenomes, use this parameter to give the ID for the reference. This is then used to build the full paths for all required reference genome files e.g. `--genome GRCh38`. \n\nSee the [nf-core website docs](https://nf-co.re/usage/reference_genomes) for more details." }, - "bowtie2": { "type": "string", "fa_icon": "fas fa-book", "description": "Path to bowtie2 index" }, - "gtf": { "type": "string", "fa_icon": "fas fa-book", "description": "Path to GTF annotation file" }, - "gene_bed": { "type": "string", "fa_icon": "fas fa-book", "description": "Path to gene BED file" }, - "blacklist": { "type": "string", "fa_icon": "fas fa-book", "description": "Path to genome blacklist" }, + "bowtie2": { + "type": "string", + "fa_icon": "fas fa-book", + "description": "Path to bowtie2 index" + }, + "gtf": { + "type": "string", + "fa_icon": "fas fa-book", + "description": "Path to GTF annotation file" + }, + "gene_bed": { + "type": "string", + "fa_icon": "fas fa-book", + "description": "Path to gene BED file" + }, + "blacklist": { + "type": "string", + "fa_icon": "fas fa-book", + "description": "Path to genome blacklist" + }, "spikein_genome": { "type": "string", "default": "K12-MG1655", @@ -164,14 +185,25 @@ "fa_icon": "fas fa-random", "description": "Skips fastqc reporting" }, - "skip_trimming": { "type": "boolean", "fa_icon": "fas fa-random", "description": "Skips trimming" }, + "skip_trimming": { + "type": "boolean", + "fa_icon": "fas fa-random", + "description": "Skips trimming" + }, "skip_removeduplicates": { "type": "boolean", "fa_icon": "fas fa-random", "description": "Skips de-duplication" }, - "skip_reporting": { "type": "boolean", "fa_icon": "fas fa-random", "description": "Skips reporting" }, - "skip_preseq": { "type": "boolean", "description": "Skips preseq reporting" }, + "skip_reporting": { + "type": "boolean", + "fa_icon": "fas fa-random", + "description": "Skips reporting" + }, + "skip_preseq": { + "type": "boolean", + "description": "Skips preseq reporting" + }, "skip_igv": { "type": "boolean", "fa_icon": "fas fa-random", @@ -192,7 +224,11 @@ "description": "Skips peak QC reporting", "fa_icon": "fas fa-random" }, - "skip_multiqc": { "type": "boolean", "fa_icon": "fas fa-random", "description": "Skips multiqc" } + "skip_multiqc": { + "type": "boolean", + "fa_icon": "fas fa-random", + "description": "Skips multiqc" + } }, "fa_icon": "fas fa-exchange-alt" }, @@ -258,13 +294,12 @@ }, "minimum_alignment_q_score": { "type": "integer", - "default": 0, + "default": 20, "fa_icon": "fas fa-sort-numeric-down", "description": "Filter reads below a q-score threshold" }, "remove_mitochondrial_reads": { "type": "boolean", - "default": false, "fa_icon": "fas fa-clone", "description": "Filter mitochondrial reads" }, @@ -281,7 +316,6 @@ "remove_linear_duplicates": { "type": "boolean", "fa_icon": "fas fa-clone", - "default": false, "description": "De-duplicate reads based on read 1 5' start position. Relevant for assays using linear amplification with tagmentation (default is false)." }, "end_to_end": { @@ -299,7 +333,7 @@ }, "normalisation_binsize": { "type": "integer", - "default": 1, + "default": 50, "description": "If normsalisation option is one of \"RPKM\", \"CPM\", \"BPM\" - then the binsize that the reads count is calculated on is used.", "fa_icon": "fas fa-arrows-alt-h" }, @@ -360,7 +394,7 @@ }, "macs_gsize": { "type": "number", - "default": 2700000000.0, + "default": 2700000000, "fa_icon": "fas fa-align-justify", "description": "parameter required by MACS2. If using an iGenomes reference these have been provided when `--genome` is set as *GRCh37*, *GRCh38*, *GRCm38*, *WBcel235*, *BDGP6*, *R64-1-1*, *EF2*, *hg38*, *hg19* and *mm10*. Otherwise the gsize will default to GRCh38." }, @@ -385,7 +419,7 @@ }, "replicate_threshold": { "type": "number", - "default": 1.0, + "default": 1, "fa_icon": "fas fa-align-justify", "description": "Minimum number of overlapping replicates needed for a consensus peak" }, @@ -462,6 +496,11 @@ "default": 0.2, "description": "Minimum peak overlap for peak reproducibility plot", "fa_icon": "fas fa-align-justify" + }, + "igv_sort_by_groups": { + "type": "boolean", + "default": true, + "description": "Sort the IGV output tracks by group" } } }, @@ -559,14 +598,12 @@ "type": "boolean", "description": "Display help text.", "fa_icon": "fas fa-question-circle", - "default": false, "hidden": true }, "version": { "type": "boolean", "description": "Display version and exit.", "fa_icon": "fas fa-question-circle", - "default": false, "hidden": true }, "publish_dir_mode": { @@ -594,7 +631,6 @@ "type": "boolean", "description": "Send plain-text email instead of HTML.", "fa_icon": "fas fa-remove-format", - "default": false, "hidden": true }, "max_multiqc_email_size": { @@ -609,7 +645,6 @@ "type": "boolean", "description": "Do not use coloured log outputs.", "fa_icon": "fas fa-palette", - "default": false, "hidden": true }, "hook_url": { @@ -648,7 +683,6 @@ "type": "boolean", "fa_icon": "far fa-eye-slash", "description": "Show all params when using `--help`", - "default": false, "hidden": true, "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." }, @@ -670,14 +704,32 @@ } }, "allOf": [ - { "$ref": "#/definitions/input_output_options" }, - { "$ref": "#/definitions/reference_data_options" }, - { "$ref": "#/definitions/flow_switching_options" }, - { "$ref": "#/definitions/trimming_options" }, - { "$ref": "#/definitions/pipeline_options" }, - { "$ref": "#/definitions/reporting_options" }, - { "$ref": "#/definitions/institutional_config_options" }, - { "$ref": "#/definitions/max_job_request_options" }, - { "$ref": "#/definitions/generic_options" } + { + "$ref": "#/definitions/input_output_options" + }, + { + "$ref": "#/definitions/reference_data_options" + }, + { + "$ref": "#/definitions/flow_switching_options" + }, + { + "$ref": "#/definitions/trimming_options" + }, + { + "$ref": "#/definitions/pipeline_options" + }, + { + "$ref": "#/definitions/reporting_options" + }, + { + "$ref": "#/definitions/institutional_config_options" + }, + { + "$ref": "#/definitions/max_job_request_options" + }, + { + "$ref": "#/definitions/generic_options" + } ] } diff --git a/pyproject.toml b/pyproject.toml index 0d62beb6..7d08e1c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,13 @@ -# Config file for Python. Mostly used to configure linting of bin/check_samplesheet.py with Black. +# Config file for Python. Mostly used to configure linting of bin/*.py with Ruff. # Should be kept the same as nf-core/tools to avoid fighting with template synchronisation. -[tool.black] +[tool.ruff] line-length = 120 -target_version = ["py37", "py38", "py39", "py310"] +target-version = "py38" +select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] +cache-dir = "~/.cache/ruff" -[tool.isort] -profile = "black" -known_first_party = ["nf_core"] -multi_line_output = 3 +[tool.ruff.isort] +known-first-party = ["nf_core"] + +[tool.ruff.per-file-ignores] +"__init__.py" = ["E402", "F401"] diff --git a/subworkflows/local/align_bowtie2.nf b/subworkflows/local/align_bowtie2.nf index 9a87cd72..8b853497 100644 --- a/subworkflows/local/align_bowtie2.nf +++ b/subworkflows/local/align_bowtie2.nf @@ -40,19 +40,21 @@ workflow ALIGN_BOWTIE2 { params.save_unaligned, false ) + ch_versions = ch_versions.mix(BOWTIE2_SPIKEIN_ALIGN.out.versions) /* * Sort, index BAM file and run samtools stats, flagstat and idxstats */ - BAM_SORT_STATS_SAMTOOLS ( BOWTIE2_TARGET_ALIGN.out.aligned, fasta ) + BAM_SORT_STATS_SAMTOOLS ( BOWTIE2_TARGET_ALIGN.out.aligned, fasta ) ch_versions = ch_versions.mix(BAM_SORT_STATS_SAMTOOLS.out.versions) - BAM_SORT_STATS_SAMTOOLS_SPIKEIN ( BOWTIE2_SPIKEIN_ALIGN.out.aligned, spikein_fasta ) + BAM_SORT_STATS_SAMTOOLS_SPIKEIN ( BOWTIE2_SPIKEIN_ALIGN.out.aligned, spikein_fasta ) + ch_versions = ch_versions.mix(BAM_SORT_STATS_SAMTOOLS_SPIKEIN.out.versions) emit: versions = ch_versions // channel: [ versions.yml ] - orig_bam = BOWTIE2_TARGET_ALIGN.out.aligned // channel: [ val(meta), bam ] + orig_bam = BOWTIE2_TARGET_ALIGN.out.aligned // channel: [ val(meta), bam ] orig_spikein_bam = BOWTIE2_SPIKEIN_ALIGN.out.aligned // channel: [ val(meta), bam ] bowtie2_log = BOWTIE2_TARGET_ALIGN.out.log // channel: [ val(meta), log_final ] diff --git a/subworkflows/local/deduplicate_linear.nf b/subworkflows/local/deduplicate_linear.nf index 4db450e1..7f27d728 100644 --- a/subworkflows/local/deduplicate_linear.nf +++ b/subworkflows/local/deduplicate_linear.nf @@ -31,11 +31,13 @@ workflow DEDUPLICATE_LINEAR { SAMTOOLS_SORT ( bam ) + ch_versions = ch_versions.mix( SAMTOOLS_SORT.out.versions ) // Convert .bam files to bed to find unique Tn5-ME-A insertion sites BEDTOOLS_BAMTOBED ( SAMTOOLS_SORT.out.bam ) + ch_versions = ch_versions.mix( BEDTOOLS_BAMTOBED.out.versions ) // Use custom .py script to find names of unique alignments FIND_UNIQUE_READS ( @@ -65,9 +67,8 @@ workflow DEDUPLICATE_LINEAR { fasta, ch_txt ) - - // Return the filtered bam in the channel - ch_bam = SAMTOOLS_VIEW.out.bam + ch_versions = ch_versions.mix( SAMTOOLS_VIEW.out.versions ) + ch_bam = SAMTOOLS_VIEW.out.bam } else { // Split out control files and run only on these @@ -83,11 +84,13 @@ workflow DEDUPLICATE_LINEAR { SAMTOOLS_SORT ( ch_split.control ) + ch_versions = ch_versions.mix( SAMTOOLS_SORT.out.versions ) // Run .bam to bed on control files only and find unique alignments in control files BEDTOOLS_BAMTOBED ( SAMTOOLS_SORT.out.bam ) + ch_versions = ch_versions.mix( BEDTOOLS_BAMTOBED.out.versions ) // Use custom .py script to find names of unique alignments in control files only FIND_UNIQUE_READS ( @@ -98,7 +101,7 @@ workflow DEDUPLICATE_LINEAR { ch_metrics = FIND_UNIQUE_READS.out.metrics ch_versions = ch_versions.mix( FIND_UNIQUE_READS.out.versions ) - // Subset original .bam file to contain only unique alignments + // Subset original .bam file to contain only unique alignments ch_split.control .join( bai ) .join( ch_linear_duplicates ) @@ -118,8 +121,7 @@ workflow DEDUPLICATE_LINEAR { fasta, ch_txt ) - - // Return the filtered bam in the channel + ch_versions = ch_versions.mix( SAMTOOLS_VIEW.out.versions ) ch_bam = SAMTOOLS_VIEW.out.bam // Prevents issues with resume with the branch elements coming in the wrong order @@ -134,10 +136,6 @@ workflow DEDUPLICATE_LINEAR { // Return the filtered control bam file concatenated with the original target bam ch_bam = ch_sorted_targets.concat( ch_sorted_controls ) } - //ch_bam | view - - // Save versions from the SAMTOOLS VIEW process - ch_versions = ch_versions.mix( SAMTOOLS_VIEW.out.versions ) /* * WORKFLOW: Re sort and index all the bam files + calculate stats @@ -146,8 +144,6 @@ workflow DEDUPLICATE_LINEAR { ch_bam, fasta ) - - // Save versions from BAM_SORT_STATS_SAMTOOLS ch_versions = ch_versions.mix( BAM_SORT_STATS_SAMTOOLS.out.versions ) emit: diff --git a/subworkflows/local/extract_fragments.nf b/subworkflows/local/extract_fragments.nf index 11c4157d..3f94c8e2 100644 --- a/subworkflows/local/extract_fragments.nf +++ b/subworkflows/local/extract_fragments.nf @@ -19,6 +19,7 @@ workflow EXTRACT_FRAGMENTS { SAMTOOLS_SORT ( bam ) + ch_versions = ch_versions.mix(SAMTOOLS_SORT.out.versions) /* * MODULE: Convert BAM file to paired-end bed format @@ -33,7 +34,7 @@ workflow EXTRACT_FRAGMENTS { * MODULE: Keep the read pairs that are on the same chromosome and fragment length less than 1000bp. */ AWK ( - BEDTOOLS_BAMTOBED.out.bed + BEDTOOLS_BAMTOBED.out.bed ) ch_versions = ch_versions.mix(AWK.out.versions) diff --git a/subworkflows/local/extract_metadata_awk.nf b/subworkflows/local/extract_metadata_awk.nf index 29adde36..7ab4abaa 100644 --- a/subworkflows/local/extract_metadata_awk.nf +++ b/subworkflows/local/extract_metadata_awk.nf @@ -1,5 +1,5 @@ /* - * Generate table-based metadata from a summary report using AWK + * Generate table-based metadata from a summary report using AWK */ include { AWK_SCRIPT } from '../../modules/local/linux/awk_script' diff --git a/subworkflows/local/peak_qc.nf b/subworkflows/local/peak_qc.nf index d09ec5e9..fd369243 100644 --- a/subworkflows/local/peak_qc.nf +++ b/subworkflows/local/peak_qc.nf @@ -36,7 +36,6 @@ workflow PEAK_QC { .join ( flagstat.map { row -> [row[0].id, row ].flatten()} ) .map { row -> [ row[1], row[2], row[4], row[6] ]} .set { ch_frip } - //ch_frip | view /* * MODULE: Calculate frip scores for primary peaks @@ -47,7 +46,7 @@ workflow PEAK_QC { min_frip_overlap ) ch_versions = ch_versions.mix(PEAK_FRIP.out.versions) - // PEAK_FRIP.out.frip_mqc | view + // PEAK_FRIP.out.frip_mqc | view /* * MODULE: Calculate peak counts for primary peaks @@ -144,8 +143,8 @@ workflow PEAK_QC { /* * MODULE: Plot upset plots for sample peaks */ - PLOT_CONSENSUS_PEAKS ( - ch_merged_bed_sorted.ifEmpty([]) + PLOT_CONSENSUS_PEAKS ( + ch_merged_bed_sorted.ifEmpty([]) ) ch_versions = ch_versions.mix(PLOT_CONSENSUS_PEAKS.out.versions) diff --git a/subworkflows/local/prepare_genome.nf b/subworkflows/local/prepare_genome.nf index f74f2840..5abc5ce3 100644 --- a/subworkflows/local/prepare_genome.nf +++ b/subworkflows/local/prepare_genome.nf @@ -37,7 +37,7 @@ workflow PREPARE_GENOME { ch_fasta = GUNZIP_FASTA ( [ [id:"target_fasta"], params.fasta ] ).gunzip ch_versions = ch_versions.mix(GUNZIP_FASTA.out.versions) } else { - ch_fasta = Channel.from( file(params.fasta) ).map { row -> [[id:"spikein_fasta"], row] } + ch_fasta = Channel.from( file(params.fasta) ).map { row -> [[id:"fasta"], row] } } /* @@ -116,7 +116,7 @@ workflow PREPARE_GENOME { /* * Index genome fasta file */ - ch_fasta_index = SAMTOOLS_FAIDX ( ch_fasta, [[id:"spikein_fasta"], []] ).fai + ch_fasta_index = SAMTOOLS_FAIDX ( ch_fasta, [[id:"fasta"], []] ).fai ch_versions = ch_versions.mix(SAMTOOLS_FAIDX.out.versions) /* diff --git a/subworkflows/local/prepare_peakcalling.nf b/subworkflows/local/prepare_peakcalling.nf index c59dd287..f4dfac51 100644 --- a/subworkflows/local/prepare_peakcalling.nf +++ b/subworkflows/local/prepare_peakcalling.nf @@ -68,6 +68,23 @@ workflow PREPARE_PEAKCALLING { ch_bedgraph = BEDTOOLS_GENOMECOV.out.genomecov //EXAMPLE CHANNEL STRUCT: [META], BEDGRAPH] //BEDTOOLS_GENOMECOV.out.genomecov | view + + /* + * CHANNEL: Dump scale factor values + */ + if(params.dump_scale_factors) { + ch_scale_factor = ch_bam_scale_factor + .map { [it[0].id, it[0].group, it[2]] } + .toSortedList( { a, b -> a[0] <=> b[0] } ) + .map { list -> + new File('scale-factors.csv').withWriter('UTF-8') { writer -> + list.each { item -> + str = item[0] + "," + item[1] + "," + item[2] + writer.write(str + "\n") + } + } + } + } } else { /* * CHANNEL: Combine bam and bai files on id @@ -130,6 +147,23 @@ workflow PREPARE_PEAKCALLING { ch_bedgraph = DEEPTOOLS_BAMCOVERAGE.out.bedgraph // EXAMPLE CHANNEL STRUCT: [[META], BAM, BAI] //ch_bedgraph | view + + /* + * CHANNEL: Dump scale factor values + */ + if(params.dump_scale_factors) { + ch_scale_factor = ch_bam_bai_scale_factor + .map { [it[0].id, it[0].group, it[3]] } + .toSortedList( { a, b -> a[0] <=> b[0] } ) + .map { list -> + new File('scale-factors.csv').withWriter('UTF-8') { writer -> + list.each { item -> + str = item[0] + "," + item[1] + "," + item[2] + writer.write(str + "\n") + } + } + } + } } /* diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index 359bf9aa..a925e0e0 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -5,9 +5,9 @@ params { } process { - cpus = 2 - memory = 6.GB - time = 6.h + cpus = 2 + memory = 6.GB + time = 6.h } if ("$PROFILE" == "singularity") { diff --git a/workflows/cutandrun.nf b/workflows/cutandrun.nf index 10d48050..6b275c47 100644 --- a/workflows/cutandrun.nf +++ b/workflows/cutandrun.nf @@ -358,7 +358,7 @@ workflow CUTANDRUN { ch_samtools_bam, ch_samtools_bai, true, - PREPARE_GENOME.out.fasta.collect(), + PREPARE_GENOME.out.fasta.collect(), PREPARE_GENOME.out.fasta_index.collect() ) ch_samtools_bam = MARK_DUPLICATES_PICARD.out.bam @@ -483,31 +483,17 @@ workflow CUTANDRUN { * MODULE: Call peaks using SEACR with IgG control */ if('seacr' in callers) { - /* - * CHANNEL: Subset control groups - */ - ch_bedgraph_target.map{ - row -> [row[0].control_group, row] - } - .set { ch_bg_target_ctrlgrp } - //ch_bg_target_ctrlgrp | view - - ch_bedgraph_control.map{ - row -> [row[0].control_group, row] - } - .set { ch_bg_control_ctrlgrp } - //ch_bg_control_ctrlgrp | view - /* * CHANNEL: Create target/control pairings */ - // Create pairs of controls (IgG) with target samples if they are supplied - ch_bg_control_ctrlgrp.cross(ch_bg_target_ctrlgrp).map { - row -> [row[1][1][0], row[1][1][1], row[0][1][1]] + ch_bedgraph_control.map{ row -> [row[0].control_group + "_" + row[0].replicate, row] } + .cross( ch_bedgraph_target.map{ row -> [row[0].control_group, row] } ) + .map { + row -> + [ row[1][1][0], row[1][1][1], row[0][1][1] ] } - .set{ ch_bedgraph_paired } + .set { ch_bedgraph_paired } // EXAMPLE CHANNEL STRUCT: [[META], TARGET_BEDGRAPH, CONTROL_BEDGRAPH] - //ch_bedgraph_paired | view SEACR_CALLPEAK_IGG ( ch_bedgraph_paired, @@ -520,31 +506,18 @@ workflow CUTANDRUN { } if('macs2' in callers) { - /* - * CHANNEL: Split control groups - */ - ch_bam_target.map{ - row -> [row[0].control_group, row] - } - .set { ch_bam_target_ctrlgrp } - //ch_bam_target_ctrlgrp | view - - ch_bam_control.map{ - row -> [row[0].control_group, row] - } - .set { ch_bam_control_ctrlgrp } - // ch_bam_control_ctrlgrp | view - /* * CHANNEL: Create target/control pairings */ - // Create pairs of controls (IgG) with target samples if they are supplied - ch_bam_control_ctrlgrp.cross(ch_bam_target_ctrlgrp).map{ - row -> [row[1][1][0], row[1][1][1], row[0][1][1]] + ch_bam_control.map{ row -> [row[0].control_group + "_" + row[0].replicate, row] } + .cross( ch_bam_target.map{ row -> [row[0].control_group, row] } ) + .map { + row -> + [ row[1][1][0], row[1][1][1], row[0][1][1] ] } - .set{ ch_bam_paired } + .set { ch_bam_paired } // EXAMPLE CHANNEL STRUCT: [[META], TARGET_BAM, CONTROL_BAM] - // ch_bam_paired | view + //ch_bam_paired | view MACS2_CALLPEAK_IGG ( ch_bam_paired, @@ -725,7 +698,8 @@ workflow CUTANDRUN { //PREPARE_GENOME.out.gtf.collect(), ch_peaks_primary.collect{it[1]}.filter{ it -> it.size() > 1}.ifEmpty([]), ch_peaks_secondary.collect{it[1]}.filter{ it -> it.size() > 1}.ifEmpty([]), - ch_bigwig.collect{it[1]}.ifEmpty([]) + ch_bigwig.collect{it[1]}.ifEmpty([]), + params.igv_sort_by_groups ) //ch_software_versions = ch_software_versions.mix(IGV_SESSION.out.versions) } @@ -986,6 +960,13 @@ workflow.onComplete { } } +workflow.onError { + if (workflow.errorReport.contains("Process requirement exceeds available memory")) { + println("🛑 Default resources exceed availability 🛑 ") + println("💡 See here on how to configure pipeline: https://nf-co.re/docs/usage/configuration#tuning-workflow-resources 💡") + } +} + //////////////////////////////////////////////////// /* -- THE END -- */ ////////////////////////////////////////////////////