From 0cfa1b19309f82929eef1f0cba553c22cc300cc2 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Sun, 8 Oct 2023 21:20:42 -0500 Subject: [PATCH] Support multiple formats in the same run Signed-off-by: Ben Sherman --- README.md | 43 +++++++++++----- nextflow.config | 17 +++++-- .../src/main/nextflow/prov/BcoRenderer.groovy | 13 ++++- .../src/main/nextflow/prov/DagRenderer.groovy | 13 ++++- .../main/nextflow/prov/LegacyRenderer.groovy | 13 ++++- .../src/main/nextflow/prov/ProvHelper.groovy | 19 +++++++ .../main/nextflow/prov/ProvObserver.groovy | 51 +++++++------------ .../nextflow/prov/ProvObserverFactory.groovy | 23 +++++++-- .../src/main/nextflow/prov/Renderer.groovy | 2 +- 9 files changed, 134 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index b7377e6..e4da46a 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,13 @@ plugins { } prov { - enabled = true - overwrite = true - file = "${params.outdir}/manifest.json" + enabled = true + formats { + legacy { + file = 'manifest.json' + overwrite = true + } + } } ``` @@ -24,31 +28,42 @@ Finally, run your Nextflow pipeline. You do not need to modify your pipeline scr ## Configuration +*The `file`, `format`, and `overwrite` options have been deprecated since version 1.2.0. Use `formats` instead.* + The following options are available: `prov.enabled` Create the provenance report (default: `true` if plugin is loaded). -`prov.file` - -The path of the provenance report (default: `manifest.json`). +`prov.formats` -`prov.format` +Configuration scope for the desired output formats. The following formats are available: -The report format. The following formats are available: - -- `bco`: Render a [BioCompute Object](https://biocomputeobject.org/). +- `bco`: Render a [BioCompute Object](https://biocomputeobject.org/). Supports the `file` and `overwrite` options. Visit the [BCO User Guide](https://docs.biocomputeobject.org/user_guide/) to learn more about this format and how to extend it with information that isn't available to Nextflow. -- `dag`: Render the task graph as a Mermaid diagram embedded in an HTML document. +- `dag`: Render the task graph as a Mermaid diagram embedded in an HTML document. Supports the `file` and `overwrite` options. -- `legacy`: Render the legacy format originally defined in this plugin (default). +- `legacy`: Render the legacy format originally defined in this plugin (default). Supports the `file` and `overwrite` options. -`prov.overwrite` +Any number of formats can be specified, for example: -Overwrite any existing provenance report with the same name (default: `false`). +```groovy +prov { + formats { + bco { + file = 'bco.json' + overwrite = true + } + legacy { + file = 'manifest.json' + overwrite = true + } + } +} +``` `prov.patterns` diff --git a/nextflow.config b/nextflow.config index e156117..6219b1b 100644 --- a/nextflow.config +++ b/nextflow.config @@ -7,7 +7,18 @@ params { } prov { - overwrite = true - file = "${params.outdir}/bco.json" - format = 'bco' + formats { + bco { + file = "${params.outdir}/bco.json" + overwrite = true + } + dag { + file = "${params.outdir}/dag.html" + overwrite = true + } + legacy { + file = "${params.outdir}/manifest.json" + overwrite = true + } + } } diff --git a/plugins/nf-prov/src/main/nextflow/prov/BcoRenderer.groovy b/plugins/nf-prov/src/main/nextflow/prov/BcoRenderer.groovy index 3d31961..4b67e3c 100644 --- a/plugins/nf-prov/src/main/nextflow/prov/BcoRenderer.groovy +++ b/plugins/nf-prov/src/main/nextflow/prov/BcoRenderer.groovy @@ -35,11 +35,22 @@ import nextflow.util.CacheHelper @CompileStatic class BcoRenderer implements Renderer { + private Path path + + private boolean overwrite + @Delegate private PathNormalizer normalizer + BcoRenderer(Map opts) { + path = opts.file as Path + overwrite = opts.overwrite as Boolean + + ProvHelper.checkFileOverwrite(path, overwrite) + } + @Override - void render(Session session, Set tasks, Map workflowOutputs, Path path) { + void render(Session session, Set tasks, Map workflowOutputs) { // get workflow inputs final taskLookup = ProvHelper.getTaskLookup(tasks) final workflowInputs = ProvHelper.getWorkflowInputs(tasks, taskLookup) diff --git a/plugins/nf-prov/src/main/nextflow/prov/DagRenderer.groovy b/plugins/nf-prov/src/main/nextflow/prov/DagRenderer.groovy index 16c541e..b3060b1 100644 --- a/plugins/nf-prov/src/main/nextflow/prov/DagRenderer.groovy +++ b/plugins/nf-prov/src/main/nextflow/prov/DagRenderer.groovy @@ -33,11 +33,22 @@ import nextflow.util.StringUtils @CompileStatic class DagRenderer implements Renderer { + private Path path + + private boolean overwrite + @Delegate private PathNormalizer normalizer + DagRenderer(Map opts) { + path = opts.file as Path + overwrite = opts.overwrite as Boolean + + ProvHelper.checkFileOverwrite(path, overwrite) + } + @Override - void render(Session session, Set tasks, Map workflowOutputs, Path path) { + void render(Session session, Set tasks, Map workflowOutputs) { // get workflow metadata final metadata = session.workflowMetadata this.normalizer = new PathNormalizer(metadata) diff --git a/plugins/nf-prov/src/main/nextflow/prov/LegacyRenderer.groovy b/plugins/nf-prov/src/main/nextflow/prov/LegacyRenderer.groovy index bbb5a71..5a1998d 100644 --- a/plugins/nf-prov/src/main/nextflow/prov/LegacyRenderer.groovy +++ b/plugins/nf-prov/src/main/nextflow/prov/LegacyRenderer.groovy @@ -32,6 +32,17 @@ import nextflow.processor.TaskRun @CompileStatic class LegacyRenderer implements Renderer { + private Path path + + private boolean overwrite + + LegacyRenderer(Map opts) { + path = opts.file as Path + overwrite = opts.overwrite as Boolean + + ProvHelper.checkFileOverwrite(path, overwrite) + } + private static def jsonify(root) { if ( root instanceof Map ) root.collectEntries( (k, v) -> [k, jsonify(v)] ) @@ -79,7 +90,7 @@ class LegacyRenderer implements Renderer { } @Override - void render(Session session, Set tasks, Map outputs, Path path) { + void render(Session session, Set tasks, Map outputs) { // generate task manifest def tasksMap = tasks.inject([:]) { accum, task -> accum[task.id] = renderTask(task) diff --git a/plugins/nf-prov/src/main/nextflow/prov/ProvHelper.groovy b/plugins/nf-prov/src/main/nextflow/prov/ProvHelper.groovy index cf4c4be..0a3ceba 100644 --- a/plugins/nf-prov/src/main/nextflow/prov/ProvHelper.groovy +++ b/plugins/nf-prov/src/main/nextflow/prov/ProvHelper.groovy @@ -19,6 +19,8 @@ package nextflow.prov import java.nio.file.Path import groovy.transform.CompileStatic +import nextflow.exception.AbortOperationException +import nextflow.file.FileHelper import nextflow.processor.TaskRun import nextflow.script.params.FileOutParam @@ -30,6 +32,23 @@ import nextflow.script.params.FileOutParam @CompileStatic class ProvHelper { + /** + * Check whether a file already exists and throw an + * error if it cannot be overwritten. + * + * @param path + * @param overwrite + */ + static void checkFileOverwrite(Path path, boolean overwrite) { + final attrs = FileHelper.readAttributes(path) + if( attrs ) { + if( overwrite && (attrs.isDirectory() || !path.delete()) ) + throw new AbortOperationException("Unable to overwrite existing provenance file: ${path.toUriString()}") + else if( !overwrite ) + throw new AbortOperationException("Provenance file already exists: ${path.toUriString()}") + } + } + /** * Get the list of output files for a task. * diff --git a/plugins/nf-prov/src/main/nextflow/prov/ProvObserver.groovy b/plugins/nf-prov/src/main/nextflow/prov/ProvObserver.groovy index 59390c8..658de6d 100644 --- a/plugins/nf-prov/src/main/nextflow/prov/ProvObserver.groovy +++ b/plugins/nf-prov/src/main/nextflow/prov/ProvObserver.groovy @@ -23,12 +23,10 @@ import java.nio.file.PathMatcher import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.Session -import nextflow.trace.TraceObserver -import nextflow.trace.TraceRecord -import nextflow.file.FileHelper import nextflow.processor.TaskHandler import nextflow.processor.TaskRun -import nextflow.exception.AbortOperationException +import nextflow.trace.TraceObserver +import nextflow.trace.TraceRecord /** * Plugin observer of workflow events @@ -40,17 +38,11 @@ import nextflow.exception.AbortOperationException @CompileStatic class ProvObserver implements TraceObserver { - public static final String DEF_FILE_NAME = 'manifest.json' - public static final List VALID_FORMATS = ['bco', 'dag', 'legacy'] private Session session - private Path path - - private Renderer renderer - - private Boolean overwrite + private List renderers private List matchers @@ -58,24 +50,22 @@ class ProvObserver implements TraceObserver { private Map workflowOutputs = [:] - ProvObserver(Path path, String format, Boolean overwrite, List patterns) { - this.path = path - this.renderer = createRenderer(format) - this.overwrite = overwrite - this.matchers = patterns.collect { pattern -> + ProvObserver(Map formats, List patterns) { + this.renderers = formats.collect( (name, config) -> createRenderer(name, config) ) + this.matchers = patterns.collect( pattern -> FileSystems.getDefault().getPathMatcher("glob:**/${pattern}") - } + ) } - private Renderer createRenderer(String format) { - if( format == 'bco' ) - return new BcoRenderer() + private Renderer createRenderer(String name, Map opts) { + if( name == 'bco' ) + return new BcoRenderer(opts) - if( format == 'dag' ) - return new DagRenderer() + if( name == 'dag' ) + return new DagRenderer(opts) - if( format == 'legacy' ) - return new LegacyRenderer() + if( name == 'legacy' ) + return new LegacyRenderer(opts) throw new IllegalArgumentException("Invalid provenance format -- valid formats are ${VALID_FORMATS.join(', ')}") } @@ -83,15 +73,6 @@ class ProvObserver implements TraceObserver { @Override void onFlowCreate(Session session) { this.session = session - - // check file existance - final attrs = FileHelper.readAttributes(path) - if( attrs ) { - if( overwrite && (attrs.isDirectory() || !path.delete()) ) - throw new AbortOperationException("Unable to overwrite existing provenance manifest: ${path.toUriString()}") - else if( !overwrite ) - throw new AbortOperationException("Provenance manifest already exists: ${path.toUriString()}") - } } @Override @@ -126,7 +107,9 @@ class ProvObserver implements TraceObserver { if( !session.isSuccess() ) return - renderer.render(session, tasks, workflowOutputs, path) + renderers.each( renderer -> + renderer.render(session, tasks, workflowOutputs) + ) } } diff --git a/plugins/nf-prov/src/main/nextflow/prov/ProvObserverFactory.groovy b/plugins/nf-prov/src/main/nextflow/prov/ProvObserverFactory.groovy index 6a9f621..c05314c 100644 --- a/plugins/nf-prov/src/main/nextflow/prov/ProvObserverFactory.groovy +++ b/plugins/nf-prov/src/main/nextflow/prov/ProvObserverFactory.groovy @@ -19,7 +19,9 @@ package nextflow.prov import java.nio.file.Path import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j import nextflow.Session +import nextflow.exception.AbortOperationException import nextflow.trace.TraceObserver import nextflow.trace.TraceObserverFactory @@ -28,6 +30,7 @@ import nextflow.trace.TraceObserverFactory * * @author Ben Sherman */ +@Slf4j @CompileStatic class ProvObserverFactory implements TraceObserverFactory { @@ -41,11 +44,21 @@ class ProvObserverFactory implements TraceObserverFactory { if( !enabled ) return - final file = config.navigate('prov.file', ProvObserver.DEF_FILE_NAME) - final path = (file as Path).complete() - final format = config.navigate('prov.format', 'legacy') as String + final format = config.navigate('prov.format') as String + final file = config.navigate('prov.file', 'manifest.json') as String final overwrite = config.navigate('prov.overwrite') as Boolean - final patterns = config.navigate('prov.patterns', []) as List - new ProvObserver(path, format, overwrite, patterns) + def formats = [:] + if( format ) { + log.warn "Config options `prov.format`, `prov.file`, and `prov.overwrite` are deprecated -- use `prov.formats` instead" + formats[format] = [file: file, overwrite: overwrite] + } + + formats = config.navigate('prov.formats', formats) as Map + + if( !formats ) + throw new AbortOperationException("Config setting `prov.formats` is required to specify provenance output formats") + + final patterns = config.navigate('prov.patterns', []) as List + new ProvObserver(formats, patterns) } } diff --git a/plugins/nf-prov/src/main/nextflow/prov/Renderer.groovy b/plugins/nf-prov/src/main/nextflow/prov/Renderer.groovy index a9a12a8..d53ea2f 100644 --- a/plugins/nf-prov/src/main/nextflow/prov/Renderer.groovy +++ b/plugins/nf-prov/src/main/nextflow/prov/Renderer.groovy @@ -27,6 +27,6 @@ import nextflow.processor.TaskRun */ interface Renderer { - abstract void render(Session session, Set tasks, Map outputs, Path path) + abstract void render(Session session, Set tasks, Map outputs) }