From a3e87febb28bd0461c22a917c5b2c1492053ef85 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 20 Aug 2024 11:34:47 +0200 Subject: [PATCH 1/5] add nfschema plugin subwf --- .../nf-core/utils_nfschema_plugin/main.nf | 32 +++++++ .../nf-core/utils_nfschema_plugin/meta.yml | 22 +++++ .../utils_nfschema_plugin/tests/main.nf.test | 62 ++++++++++++ .../tests/nextflow.config | 8 ++ .../tests/nextflow_schema.json | 96 +++++++++++++++++++ 5 files changed, 220 insertions(+) create mode 100644 subworkflows/nf-core/utils_nfschema_plugin/main.nf create mode 100644 subworkflows/nf-core/utils_nfschema_plugin/meta.yml create mode 100644 subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test create mode 100644 subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config create mode 100644 subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json diff --git a/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/subworkflows/nf-core/utils_nfschema_plugin/main.nf new file mode 100644 index 00000000000..5d1dc7c9282 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -0,0 +1,32 @@ +// +// Subworkflow that uses the nf-schema plugin to validate parameters and render the parameter summary +// + +include { paramsSummaryLog } from 'plugin/nf-schema' +include { validateParameters } from 'plugin/nf-schema' + +workflow UTILS_NFSCHEMA_PLUGIN { + + take: + validate_params // boolean: validate the parameters + + main: + + // + // Print parameter summary to stdout. This will display the parameters + // that differ from the default given in the JSON schema + // + log.info paramsSummaryLog(workflow) + + // + // Validate the parameters using nextflow_schema.json or the schema + // given via the validation.parametersSchema configuration option + // + if(validate_params) { + validateParameters() + } + + emit: + dummy_emit = true +} + diff --git a/subworkflows/nf-core/utils_nfschema_plugin/meta.yml b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml new file mode 100644 index 00000000000..90b8cb18982 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "utils_nfschema_plugin" +description: Run nf-schema to validate parameters and create a summary of changed parameters +keywords: + - validation + - JSON schema + - plugin + - parameters + - summary +components: [] +input: + - validate_params: + type: boolean + description: Validate the parameters and error if invalid. +output: + - dummy_emit: + type: boolean + description: Dummy emit to make nf-core subworkflows lint happy +authors: + - "@nvnieuwk" +maintainers: + - "@nvnieuwk" diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test new file mode 100644 index 00000000000..703d3a9b86a --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test @@ -0,0 +1,62 @@ +// TODO nf-core: Once you have added the required tests, please run the following command to build this file: +// nf-core subworkflows test utils_nfschema_plugin +nextflow_workflow { + + name "Test Subworkflow UTILS_NFSCHEMA_PLUGIN" + script "../main.nf" + workflow "UTILS_NFSCHEMA_PLUGIN" + + tag "subworkflows" + tag "subworkflows_nfcore" + tag "subworkflows/utils_nfschema_plugin" + + config "./nextflow.config" + + test("Should run nothing") { + + when { + + params { + test_data = '' + } + + workflow { + """ + validate_params = false + input[0] = validate_params + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should validate params") { + + when { + + params { + test_data = '' + outdir = 1 + } + + workflow { + """ + validate_params = true + input[0] = validate_params + """ + } + } + + then { + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ Validation of pipeline parameters failed!') } } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config new file mode 100644 index 00000000000..0907ac58f00 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config @@ -0,0 +1,8 @@ +plugins { + id "nf-schema@2.1.0" +} + +validation { + parametersSchema = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + monochromeLogs = true +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json new file mode 100644 index 00000000000..331e0d2f443 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/./master/nextflow_schema.json", + "title": ". pipeline parameters", + "description": "", + "type": "object", + "$defs": { + "input_output_options": { + "title": "Input/output options", + "type": "object", + "fa_icon": "fas fa-terminal", + "description": "Define where the pipeline should find input data and save output data.", + "required": ["outdir"], + "properties": { + "validate_params": { + "type": "boolean", + "description": "Validate parameters?", + "default": true, + "hidden": true + }, + "outdir": { + "type": "string", + "format": "directory-path", + "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", + "fa_icon": "fas fa-folder-open" + }, + "test_data_base": { + "type": "string", + "default": "https://raw.githubusercontent.com/nf-core/test-datasets/modules", + "description": "Base for test data directory", + "hidden": true + }, + "test_data": { + "type": "string", + "description": "Fake test data param", + "hidden": true + } + } + }, + "generic_options": { + "title": "Generic options", + "type": "object", + "fa_icon": "fas fa-file-import", + "description": "Less common options for the pipeline, typically set in a config file.", + "help_text": "These options are common to all nf-core pipelines and allow you to customise some of the core preferences for how the pipeline runs.\n\nTypically these options would be set in a Nextflow config file loaded for all pipeline runs, such as `~/.nextflow/config`.", + "properties": { + "help": { + "type": "boolean", + "description": "Display help text.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, + "version": { + "type": "boolean", + "description": "Display version and exit.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, + "logo": { + "type": "boolean", + "default": true, + "description": "Display nf-core logo in console output.", + "fa_icon": "fas fa-image", + "hidden": true + }, + "singularity_pull_docker_container": { + "type": "boolean", + "description": "Pull Singularity container from Docker?", + "hidden": true + }, + "publish_dir_mode": { + "type": "string", + "default": "copy", + "description": "Method used to save pipeline results to output directory.", + "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", + "fa_icon": "fas fa-copy", + "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], + "hidden": true + }, + "monochrome_logs": { + "type": "boolean", + "description": "Use monochrome_logs", + "hidden": true + } + } + } + }, + "allOf": [ + { + "$ref": "#/$defs/input_output_options" + }, + { + "$ref": "#/$defs/generic_options" + } + ] +} From 1239f69efcdb6a7938ecb0cdbc712425e5a53a87 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 20 Aug 2024 15:35:03 +0200 Subject: [PATCH 2/5] fix nf-test version and nf-core lint --- .github/workflows/test.yml | 2 ++ subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 760d9136e47..89a74294bcf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -646,6 +646,8 @@ jobs: - name: Install nf-test uses: nf-core/setup-nf-test@v1 + with: + version: ${{ env.NFTEST_VER }} - name: Setup apptainer if: matrix.profile == 'singularity' diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test index 703d3a9b86a..643eba58c09 100644 --- a/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test @@ -1,5 +1,3 @@ -// TODO nf-core: Once you have added the required tests, please run the following command to build this file: -// nf-core subworkflows test utils_nfschema_plugin nextflow_workflow { name "Test Subworkflow UTILS_NFSCHEMA_PLUGIN" @@ -9,6 +7,7 @@ nextflow_workflow { tag "subworkflows" tag "subworkflows_nfcore" tag "subworkflows/utils_nfschema_plugin" + tag "plugin/nf-schema" config "./nextflow.config" From c27172ddb474c7b2f8612eeb54508540f0cece0b Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 23 Aug 2024 10:10:23 +0200 Subject: [PATCH 3/5] adjust to review comments --- .../nf-core/utils_nfschema_plugin/main.nf | 20 ++++++- .../nf-core/utils_nfschema_plugin/meta.yml | 13 ++++ .../utils_nfschema_plugin/tests/main.nf.test | 60 ++++++++++++++++++- 3 files changed, 88 insertions(+), 5 deletions(-) diff --git a/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/subworkflows/nf-core/utils_nfschema_plugin/main.nf index 5d1dc7c9282..4f1615eac75 100644 --- a/subworkflows/nf-core/utils_nfschema_plugin/main.nf +++ b/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -8,7 +8,13 @@ include { validateParameters } from 'plugin/nf-schema' workflow UTILS_NFSCHEMA_PLUGIN { take: - validate_params // boolean: validate the parameters + input_workflow // workflow: the workflow object used by nf-schema to get metadata from the workflow + validate_params // boolean: validate the parameters + parameters_schema // string: path to the parameters JSON schema. + // this has to be the same as the schema given to `validation.parametersSchema` + // when this input is empty it will automatically use the configured schema or + // "${projectDir}/nextflow_schema.json" as default. This input should not be empty + // for meta pipelines main: @@ -16,14 +22,22 @@ workflow UTILS_NFSCHEMA_PLUGIN { // Print parameter summary to stdout. This will display the parameters // that differ from the default given in the JSON schema // - log.info paramsSummaryLog(workflow) + if(parameters_schema) { + log.info paramsSummaryLog(input_workflow, parameters_schema:parameters_schema) + } else { + log.info paramsSummaryLog(input_workflow) + } // // Validate the parameters using nextflow_schema.json or the schema // given via the validation.parametersSchema configuration option // if(validate_params) { - validateParameters() + if(parameters_schema) { + validateParameters(parameters_schema:parameters_schema) + } else { + validateParameters() + } } emit: diff --git a/subworkflows/nf-core/utils_nfschema_plugin/meta.yml b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml index 90b8cb18982..5626af30011 100644 --- a/subworkflows/nf-core/utils_nfschema_plugin/meta.yml +++ b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml @@ -9,9 +9,22 @@ keywords: - summary components: [] input: + - input_workflow: + type: object + description: | + The workflow object of the used pipeline. + This object contains meta data used to create the params summary log - validate_params: type: boolean description: Validate the parameters and error if invalid. + - parameters_schema: + type: string + description: | + Path to the parameters JSON schema. + This has to be the same as the schema given to the `validation.parametersSchema` config + option. When this input is empty it will automatically use the configured schema or + "${projectDir}/nextflow_schema.json" as default. The schema should not be given in this way + for meta pipelines. output: - dummy_emit: type: boolean diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test index 643eba58c09..6b519935144 100644 --- a/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test @@ -22,7 +22,9 @@ nextflow_workflow { workflow { """ validate_params = false - input[0] = validate_params + input[0] = workflow + input[1] = validate_params + input[2] = "" """ } } @@ -46,7 +48,61 @@ nextflow_workflow { workflow { """ validate_params = true - input[0] = validate_params + input[0] = workflow + input[1] = validate_params + input[2] = "" + """ + } + } + + then { + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ Validation of pipeline parameters failed!') } } + ) + } + } + + test("Should run nothing - custom schema") { + + when { + + params { + test_data = '' + } + + workflow { + """ + validate_params = false + input[0] = workflow + input[1] = validate_params + input[2] = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should validate params") { + + when { + + params { + test_data = '' + outdir = 1 + } + + workflow { + """ + validate_params = true + input[0] = workflow + input[1] = validate_params + input[2] = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" """ } } From 2506a88fc51a6fe5605d2be3a6c1ab36e356fc9c Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 23 Aug 2024 10:13:08 +0200 Subject: [PATCH 4/5] update test name --- subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test index 6b519935144..842dc432af7 100644 --- a/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test @@ -88,7 +88,7 @@ nextflow_workflow { } } - test("Should validate params") { + test("Should validate params - custom schema") { when { From 63af4c55476093d352d7610c1c1ca0bacadde361 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 23 Aug 2024 10:46:18 +0200 Subject: [PATCH 5/5] appease linting --- subworkflows/nf-core/utils_nfschema_plugin/main.nf | 14 +++++++------- .../nf-core/utils_nfschema_plugin/meta.yml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/subworkflows/nf-core/utils_nfschema_plugin/main.nf index 4f1615eac75..4994303ea0b 100644 --- a/subworkflows/nf-core/utils_nfschema_plugin/main.nf +++ b/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -8,13 +8,13 @@ include { validateParameters } from 'plugin/nf-schema' workflow UTILS_NFSCHEMA_PLUGIN { take: - input_workflow // workflow: the workflow object used by nf-schema to get metadata from the workflow - validate_params // boolean: validate the parameters - parameters_schema // string: path to the parameters JSON schema. - // this has to be the same as the schema given to `validation.parametersSchema` - // when this input is empty it will automatically use the configured schema or - // "${projectDir}/nextflow_schema.json" as default. This input should not be empty - // for meta pipelines + input_workflow // workflow: the workflow object used by nf-schema to get metadata from the workflow + validate_params // boolean: validate the parameters + parameters_schema // string: path to the parameters JSON schema. + // this has to be the same as the schema given to `validation.parametersSchema` + // when this input is empty it will automatically use the configured schema or + // "${projectDir}/nextflow_schema.json" as default. This input should not be empty + // for meta pipelines main: diff --git a/subworkflows/nf-core/utils_nfschema_plugin/meta.yml b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml index 5626af30011..f7d9f028850 100644 --- a/subworkflows/nf-core/utils_nfschema_plugin/meta.yml +++ b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml @@ -20,7 +20,7 @@ input: - parameters_schema: type: string description: | - Path to the parameters JSON schema. + Path to the parameters JSON schema. This has to be the same as the schema given to the `validation.parametersSchema` config option. When this input is empty it will automatically use the configured schema or "${projectDir}/nextflow_schema.json" as default. The schema should not be given in this way