From 5d30a757abf9bc24d7f69b260bec446ae4f89ffc Mon Sep 17 00:00:00 2001 From: Kevin Lewis Date: Mon, 10 Jul 2017 10:49:27 +0100 Subject: [PATCH 01/20] Select directive and splice prune pv (#186) * vtfp.pl - correct logic for allowing edges without id attributes * add tests for splicing and pruning from graphs which have edge elements without id values * add select directive * initial tests for select directive * add more tests for vtfp select directive * fixes for select directive * changes to names of temporary output files (previous were potentially misleading results of cut-and-paste) * minor tweaks and a couple of new tests * amend splice/prune processing to allow these operations to be specified in a param_vals file * vtfp.pl: bug fixes for select directive more and improved tests * add select directive * initial tests for select directive * add more tests for vtfp select directive * fixes for select directive * changes to names of temporary output files (previous were potentially misleading results of cut-and-paste) * minor tweaks and a couple of new tests * add tests for splice/prune via param_vals file --- bin/vtfp.pl | 394 ++++++++++++---- t/10-vtfp-pv.t | 155 +++++- t/10-vtfp-select_directive.t | 884 +++++++++++++++++++++++++++++++++++ t/10-vtfp-splice_nodes.t | 3 +- t/10-vtfp-subst_directive.t | 6 +- 5 files changed, 1350 insertions(+), 92 deletions(-) create mode 100644 t/10-vtfp-select_directive.t diff --git a/bin/vtfp.pl b/bin/vtfp.pl index 81632c7e0..a68c6579a 100755 --- a/bin/vtfp.pl +++ b/bin/vtfp.pl @@ -73,7 +73,7 @@ @vals = map { split_csl($_); } @vals; @param_vals_fns = map { split_csl($_); } @param_vals_fns; -my $params = initialise_params(\@keys, \@vals, \@nullkeys, \@param_vals_fns); +my $params = initialise_params(\@keys, \@vals, \@nullkeys, $splice_list, $prune_list, \@param_vals_fns); if($export_param_vals) { open my $epv, ">$export_param_vals" or croak "Failed to open $export_param_vals for export of param_vals"; @@ -104,12 +104,11 @@ my $node_tree = process_vtnode(q[], $vtf_name, q[], $params, $globals); # recursively generate the vtnode tree - my $flat_graph = flatten_tree($node_tree); my $cull_node_ids = []; # used to detect (and disregard) parameter substitution errors in nodes which have been removed -if($splice_list or $prune_list) { - ($flat_graph, $cull_node_ids) = splice_nodes($flat_graph, $splice_list, $prune_list); +if(@{$params->{ops}->{splice}} or @{$params->{ops}->{prune}}) { + ($flat_graph, $cull_node_ids) = splice_nodes($flat_graph, $params->{ops}); } # parameter substitution errors are reported late, so errors in pruned or spliced nodes can be ignored @@ -305,39 +304,53 @@ sub process_subst_params { for my $i (0..$#{$unprocessed_subst_params}) { - my $sp = $unprocessed_subst_params->[$i]; - my $spid = $sp->{id}; - my $sptype = $sp->{type}; - $sptype ||= q[PARAM]; + my $sps = $unprocessed_subst_params->[$i]; + $sps = subst_walk($sps, $params, $ewi, { irp => [], select_opts => { select => 1, }, } ); # preprocess any select directives in the subst_params section - if($sptype eq q[SPFILE]) { # process recursively - # SPFILE entries will be processed after all PARAM-type entries have been processed (for consistency in redeclaration behaviour) - push @spfile_node_queue, $sp; + if(not defined $sps) { next; } + # subst_walk may return an array of entries, so make that always the case. + if(ref $sps ne q[ARRAY]) { + $sps = [ $sps ]; } - elsif($sptype eq q[PARAM]) { - # all unprocessed_subst_params elements of type PARAM must have an id - if(not $spid) { - # cache errors so we can report as many as possible before exit - $ewi->{additem}->($EWI_ERROR, 0, q[No id for PARAM element, entry ], $i, q[ (], , join(q[->], @$sp_file_stack), q[)]); + + for my $sp (@{$sps}) { + if(not $sp or ref $sp ne q[HASH] or not $sp->{id}) { + $ewi->{additem}->($EWI_ERROR, 0, q[Failed to resolve subst_params element ], $i, q[ (], , join(q[->], @$sp_file_stack), q[)]); + next; } + my $spid = $sp->{id}; + my $sptype = $sp->{type}; + $sptype ||= q[PARAM]; - my $ips = in_param_store($param_store, $spid); - if($ips->{errnum} != 0) { # multiply defined - OK unless explicitly declared multiple times at this level - if($ips->{errnum} > 0) { # a previous declaration was made by an ancestor of the current vtnode - $ewi->{additem}->($EWI_INFO, 2, qq[INFO: Duplicate subst_param definition for $spid (], join(q[->], @$sp_file_stack), q[); ], $ips->{ms}); - } - else { + if($sptype eq q[SPFILE]) { # process recursively + # SPFILE entries will be processed after all PARAM-type entries have been processed (for consistency in redeclaration behaviour) + push @spfile_node_queue, $sp; + } + elsif($sptype eq q[PARAM]) { + # all unprocessed_subst_params elements of type PARAM must have an id + if(not $spid) { # cache errors so we can report as many as possible before exit - $ewi->{additem}->($EWI_ERROR, 0, qq[Fatal error: Duplicate (local) subst_param definition for $spid (], join(q[->], @$sp_file_stack), q[); ], $ips->{ms}); + $ewi->{additem}->($EWI_ERROR, 0, q[No id for PARAM element, entry ], $i, q[ (], , join(q[->], @$sp_file_stack), q[)]); } - } - $sp->{_declared_by} ||= []; - push @{$sp->{_declared_by}}, join q[->], @$sp_file_stack; - $param_store->[0]->{varnames}->{$spid} = $sp; # adding to the "local" variable store - } - else { - $ewi->{additem}->($EWI_ERROR, 0, q[Unrecognised type for subst_param element: ], $sptype, q[; entry ], $i, q[ (], , join(q[->], @$sp_file_stack), q[)]); + my $ips = in_param_store($param_store, $spid); + if($ips->{errnum} != 0) { # multiply defined - OK unless explicitly declared multiple times at this level + if($ips->{errnum} > 0) { # a previous declaration was made by an ancestor of the current vtnode + $ewi->{additem}->($EWI_INFO, 2, qq[INFO: Duplicate subst_param definition for $spid (], join(q[->], @$sp_file_stack), q[); ], $ips->{ms}); + } + else { + # cache errors so we can report as many as possible before exit + $ewi->{additem}->($EWI_ERROR, 0, qq[Fatal error: Duplicate (local) subst_param definition for $spid (], join(q[->], @$sp_file_stack), q[); ], $ips->{ms}); + } + } + + $sp->{_declared_by} ||= []; + push @{$sp->{_declared_by}}, join q[->], @$sp_file_stack; + $param_store->[0]->{varnames}->{$spid} = $sp; # adding to the "local" variable store + } + else { + $ewi->{additem}->($EWI_ERROR, 0, q[Unrecognised type for subst_param element: ], $sptype, q[; entry ], $i, q[ (], , join(q[->], @$sp_file_stack), q[)]); + } } } @@ -346,7 +359,7 @@ sub process_subst_params { ################################ for my $spfile (@spfile_node_queue) { my $ewi = mkewi(q[SPF]); - $spfile = subst_walk($spfile, $params, $ewi, []); + $spfile = subst_walk($spfile, $params, $ewi); my $spname = is_valid_name($spfile->{name}); if(not $spname) { # cache these errors and report as many as possible before exit @@ -416,7 +429,7 @@ sub apply_subst { $ewi->{settag}->(\$id); $ewi->{addlabel}->(q{assigning to id:[} . $elem->{id} . q{]}); - $elem = subst_walk($elem, $params, $ewi, []); + $elem = subst_walk($elem, $params, $ewi); $id = $vtnode_prefix . (exists $elem->{id} and $elem->{id})? $elem->{id} : q[NOID]; $ewi->{removelabel}->(); @@ -427,7 +440,7 @@ sub apply_subst { my $id = $elem->{id}; $id ||= q[NOID]; $ewi->{addlabel}->(q{assigning to id:[} . $id . q{]}); - $elem = subst_walk($elem, $params, $ewi, []); + $elem = subst_walk($elem, $params, $ewi); $ewi->{removelabel}->(); } @@ -441,10 +454,10 @@ sub apply_subst { # to caller ################################################################################################## sub subst_walk { - my ($elem, $params, $ewi, $irp) = @_; + my ($elem, $params, $ewi, $aux) = @_; - # first check if elem itself needs resolution (is itself a subst, subst_constructor or select directive) - $elem = fetch_sp_value($elem, $params, $ewi, $irp); + # first check if elem itself needs resolution (is itself a subst/subst_constructor/select directive) + $elem = fetch_sp_value($elem, $params, $ewi, $aux); my $r = ref $elem; if(!$r) { @@ -452,19 +465,19 @@ sub subst_walk { } elsif(ref $elem eq q[HASH]) { for my $k (keys %$elem) { - $elem->{$k} = fetch_sp_value($elem->{$k}, $params, $ewi, $irp); + $elem->{$k} = fetch_sp_value($elem->{$k}, $params, $ewi, $aux); if(defined $elem->{$k}) { - $elem->{$k} = subst_walk($elem->{$k}, $params, $ewi, $irp); + $elem->{$k} = subst_walk($elem->{$k}, $params, $ewi, $aux); } } } elsif(ref $elem eq q[ARRAY]) { for my $i (reverse (0 .. $#{$elem})) { - $elem->[$i] = fetch_sp_value($elem->[$i], $params, $ewi, $irp); + $elem->[$i] = fetch_sp_value($elem->[$i], $params, $ewi, $aux); if(defined $elem->[$i]) { - $elem->[$i] = subst_walk($elem->[$i], $params, $ewi, $irp); + $elem->[$i] = subst_walk($elem->[$i], $params, $ewi, $aux); } } } @@ -483,19 +496,28 @@ sub subst_walk { # directive). If so pass it to the appropriate resolver and return the result ############################################################################################# sub fetch_sp_value { - my ($elem, $params, $ewi, $irp) = @_; + my ($elem, $params, $ewi, $aux) = @_; my $retval; + my $select_opts = { subst =>1, subst_constructor => 1, select => 1, }; # default to parse all directives; + if($aux and defined $aux->{select_opts}) { + $select_opts = $aux->{select_opts} + } + my $ref_type = ref $elem; if($ref_type) { if($ref_type eq q[HASH]) { - if($elem->{subst}) { + if($select_opts->{subst} and $elem->{subst}) { # subst directive - $retval = resolve_subst($elem, $params, $ewi, $irp); + $retval = resolve_subst($elem, $params, $ewi, $aux); } - elsif($elem->{subst_constructor}) { + elsif($select_opts->{subst_constructor} and $elem->{subst_constructor}) { # solo subst_constructor - $retval = resolve_subst_constructor(q[ANON], $elem->{subst_constructor}, $params, $ewi); + $retval = resolve_subst_constructor(q[ANON], $elem->{subst_constructor}, $params, $ewi, $aux); + } + elsif($select_opts->{select} and $elem->{select}) { + # select directive + $retval = resolve_select_value($elem, $params, $ewi, $aux); } else { $retval = $elem; @@ -536,12 +558,12 @@ sub fetch_sp_value { # flags or import_params mechanism). ############################################################ sub resolve_subst { - my ($subst, $params, $ewi, $irp) = @_; + my ($subst, $params, $ewi, $aux) = @_; my $retval; # check to see if an sp_expr needs evaluating if(ref $subst->{subst}) { # subst name is itself an expression which needs evaluation - $subst->{subst} = subst_walk($subst->{subst}, $params, $ewi, $irp); + $subst->{subst} = subst_walk($subst->{subst}, $params, $ewi, $aux); } if(ref $subst->{subst}) { # TODO - consider implications of allowing an array here @@ -554,7 +576,7 @@ sub resolve_subst { ################################################## # fetch the param_store element for the param_name ################################################## - my $param_entry = fetch_param_entry($param_name, $params, $ewi, $irp); + my $param_entry = fetch_param_entry($param_name, $params, $ewi, $aux); if(exists $param_entry->{_value}) { # allow undef value if(not defined $param_entry->{_value} and $subst->{required} and $subst->{required} !~ /\A(false|no|off)\Z/i) { # explicitly set to undef $ewi->{additem}->($EWI_ERROR, 0, q[Undef value specified for required subst (param_name: ], $param_name, q[)]); @@ -565,7 +587,7 @@ sub resolve_subst { ####################################################################################################### # no value retrieved from param_store, try to construct one from the subst directive's ifnull attribute ####################################################################################################### - if($retval = resolve_ifnull($param_name, $subst->{ifnull}, $params, $ewi, $irp)) { + if($retval = resolve_ifnull($param_name, $subst->{ifnull}, $params, $ewi, $aux)) { return $retval; # note: result of ifnull evaluation not assigned to variable } elsif($subst->{required} and $subst->{required} and $subst->{required} !~ /\A(false|no|off)\Z/i) { @@ -573,8 +595,6 @@ sub resolve_subst { return; } - $param_entry->{_value} = $retval; - return $retval; } @@ -594,12 +614,14 @@ sub resolve_subst { # requested param_name. ################################################################### sub fetch_param_entry { - my ($param_name, $params, $ewi, $irp) = @_; + my ($param_name, $params, $ewi, $aux) = @_; my $param_entry; my $retval; - if(defined $irp and any { $_ eq $param_name } @{$irp}) { # infinite recursion prevention - $ewi->{additem}->($EWI_ERROR, 0, q[infinite recursion detected resolving parameter ], $param_name, q[ (], join(q/=>/, (@{$irp}, $param_name)), q[)]); + $aux ||= { irp => [], }; + $aux->{irp} ||= []; + if(any { $_ eq $param_name} @{$aux->{irp}}) { # infinite recursion prevention + $ewi->{additem}->($EWI_ERROR, 0, q[infinite recursion detected resolving parameter ], $param_name, q[ (], join(q/=>/, (@{$aux->{irp}}, $param_name)), q[)]); return; } @@ -654,8 +676,8 @@ sub fetch_param_entry { $candidate = $param_entry; # already evaluated, return cached value (allowing undef) } - push @{$irp}, $param_name; - $retval = resolve_subst_constructor($param_name, $param_entry->{subst_constructor}, $params, $ewi, $irp); + push @{$aux->{irp}}, $param_name; + $retval = resolve_subst_constructor($param_name, $param_entry->{subst_constructor}, $params, $ewi, $aux); if(not $retval and $candidate) { $retval = $candidate->{_value}; @@ -665,25 +687,24 @@ sub fetch_param_entry { $param_entry->{_value} = $retval; } else { - $retval = resolve_param_default($param_name, $param_entry->{default}, $params, $ewi, $irp); + $retval = resolve_param_default($param_name, $param_entry->{default}, $params, $ewi, $aux); if(defined $retval) { $param_entry->{_value} = $retval; } } - pop @{$irp}; + pop @{$aux->{irp}}; if(not defined $retval) { # caller should decide if undef is allowed, unless required is true my $severity = (defined $param_entry->{required} and $param_entry->{required} and $param_entry->{required} !~ /\A(false|no|off)\Z/i)? $EWI_ERROR: $EWI_INFO; $ewi->{additem}->($severity, 0, q[No value found for param_entry ], $param_name); - return; } return $param_entry; } sub resolve_subst_constructor { - my ($id, $subst_constructor, $params, $ewi, $irp) = @_; + my ($id, $subst_constructor, $params, $ewi, $aux) = @_; if(not defined $subst_constructor) { return; } @@ -693,7 +714,7 @@ sub resolve_subst_constructor { return; } - $value = subst_walk($value, $params, $ewi, $irp); + $value = subst_walk($value, $params, $ewi, $aux); if(not defined $value) { $ewi->{additem}->($EWI_ERROR, 0, q[Error processing subst_constructor value, param_name: ], $id); @@ -779,20 +800,193 @@ sub postprocess_subst_array { return $subst_value; } +sub resolve_select_value { + my ($select, $params, $ewi, $aux) = @_; + my $param_entry; + my $retval; + + # check to see if select value is an expression which needs evaluating + my $indexes = subst_walk($select->{select}, $params, $ewi, $aux); + if(ref $indexes and ref $indexes ne q[ARRAY]) { # only scalar or array (of scalars) allowed here + $ewi->{additem}->($EWI_ERROR, 0, q[select key can only be a scalar or array, not ref (type: ], ref $indexes, q[)]); + return; + } + if(not ref $indexes) { $indexes = [ $indexes ]; } + my $id_string = join(q/;/, @{$indexes}); # for error logging + if(not defined ($indexes = _resolve_indexes($indexes, $params, $ewi, $aux))) { + return; + } + + my $cases = _preproc_cases($select->{cases}, $id_string, $params, $ewi, $aux); + if(not $cases) { + return; + } + + # this approach means that default will always apply when no select indexes are found (so cannot be switched off using -nullkeys) + if(scalar @{$indexes} == 0) { + # all valid but nothing selected - evaluate default attribute (if exists), revalidate as select_range + my $default; + if(defined $select->{default}) { + $default = subst_walk($select->{default}, $params, $ewi, $aux); + } + + if(not $default or (ref $default eq q[ARRAY] and not @{$default})) { return; } + + $indexes = $default; + if(not ref $indexes) { $indexes = [ $indexes ]; } + } + + $indexes = finalise_array($indexes); # do this after default check + + # validate indices - numerics for array cases, existing keys for hash cases + if(not defined ($indexes = _validate_indexes($indexes, $select->{cases}, $params, $ewi))) { # array indices numeric and in range? hash keys exist in hash? + $ewi->{additem}->($EWI_ERROR, 0, q[select directive without valid indexes (select on: ], $id_string, q[)]); + return; + } + + if(defined $select->{select_range} and not _valid_select_range($select->{select_range}, $indexes, $ewi, $id_string, q[select_range])) { + return; + } + + if(@{$indexes} == 0) { # check again, maybe only an array of undefs before + return; + } + + if(ref $cases eq q[ARRAY]) { + if(@{$indexes} > 1) { + $retval = [ @{$cases}[@{$indexes}] ]; + } + else { + $retval = $cases->[$indexes->[0]]; + } + } + else { + if(@{$indexes} > 1) { + $retval = [ @{$cases}{@{$indexes}} ]; # slice the hash + } + else { + $retval = $cases->{$indexes->[0]}; + } + } + + return $retval; +} + +sub _resolve_indexes { + my ($indexes, $params, $ewi, $aux) = @_; + + if(ref $indexes ne q[ARRAY]) { + $ewi->{additem}->($EWI_ERROR, 0, q[indexes attribute must be array]); + return; + } + + for my $i (0 .. $#{$indexes}) { + my $param_entry = fetch_param_entry($indexes->[$i], $params, $ewi, $aux); + if(exists $param_entry->{_value}) { + $indexes->[$i] = $param_entry->{_value}; + } + else { + splice @{$indexes}, $i, 1; + } + } + + return $indexes; +} + +sub _preproc_cases { + my ($cases, $id_string, $params, $ewi, $aux) = @_; + + if(not (defined $cases)) { + $ewi->{additem}->($EWI_ERROR, 0, q[select directive with undefine or missing cases attribute (select on: ], $id_string, q[)]); + return; + } + if(not ($cases = subst_walk($cases, $params, $ewi, $aux))) { # resolve what needs to be resolved + $ewi->{additem}->($EWI_ERROR, 0, q[select directive case atttribute resolves to undef (select on: ], $id_string, q[)]); + return; + } + my $rt = ref $cases; + if(not any { /\A$rt\Z/ } qw(ARRAY HASH)) { + $ewi->{additem}->($EWI_ERROR, 0, q[cases attribute must be array or hash (select on: ], $id_string, q[)]); + return; + } + + return $cases; +} + +sub _validate_indexes { + my ($indexes, $cases, $params, $ewi) = @_; + + if(ref $indexes ne q[ARRAY]) { + $ewi->{additem}->($EWI_ERROR, 0, q[indexes attribute must be array]); + return; # should this be fatal or just informational? + } + + my $def_indexes = [ (grep { defined } @{$indexes}) ]; # undefined values are allowed + my $cases_type = ref $cases; + my @outsiders; + if($cases_type eq q[ARRAY]) { + my $maxidx = scalar @{$cases}; + + @outsiders = grep { $_ > $maxidx; } @{$def_indexes}; + if(@outsiders) { + $ewi->{additem}->($EWI_ERROR, 0, q[index values out of range for array of cases: ], join(',', @outsiders)); + return; # should this be fatal or just informational? + } + @outsiders = grep { /\D/ } @{$def_indexes}; + if(@outsiders) { + $ewi->{additem}->($EWI_ERROR, 0, q[non-numeric index values for array of cases: ], join(',', @outsiders)); + return; # should this be fatal or just informational? + } + } + elsif($cases_type eq q[HASH]) { + @outsiders = grep { not exists $cases->{$_} } @{$def_indexes}; + if(@outsiders) { + $ewi->{additem}->($EWI_ERROR, 0, q[keys don't exist in cases: ], join(',', @outsiders)); + return; # should this be fatal or just informational? + } + } + else { + $ewi->{additem}->($EWI_ERROR, 0, q[cases attribute must be array or hash, not ], ($cases_type)? $cases_type : q[scalar]); + return; + } + + return $indexes; +} + +sub _valid_select_range { + my ($select_range, $indexes, $ewi, $id_string, $type) = @_; + + if(ref $select_range ne q[ARRAY] or @{$select_range} < 1 or @{$select_range} > 2) { + $ewi->{additem}->($EWI_ERROR, 0, q[select_range attribute of a ], $type, q[ must be an array with one or two elements (select on: ], $id_string, q[)]); + return; + } + + my ($min, $max) = sort { $a <=> $b } @{$select_range}; + $max ||= $min; + + my $n = @{$indexes}; + if($n < $min or $n > $max) { + $ewi->{additem}->($EWI_ERROR, 0, q[number of indexes in ], $type , q[ is incorrect, should be >= ], $min, q[ and <= ], $max, q[, is ], $n, q[ (select on: ], $id_string, q[)]); + return; + } + + return $select_range; +} + sub resolve_param_default { - my ($id, $default, $params, $ewi, $irp) = @_; + my ($id, $default, $params, $ewi, $aux) = @_; if(not defined $default) { return; } - return subst_walk($default, $params, $ewi, $irp); + return subst_walk($default, $params, $ewi, $aux); } sub resolve_ifnull { - my ($id, $ifnull, $params, $ewi, $irp) = @_; + my ($id, $ifnull, $params, $ewi, $aux) = @_; if(not defined $ifnull) { return; } - return subst_walk($ifnull, $params, $ewi, $irp); + return subst_walk($ifnull, $params, $ewi, $aux); } @@ -965,13 +1159,12 @@ sub get_child_prefix { sub splice_nodes { - my ($flat_graph, $splice_list, $prune_list) = @_; - my $splice_candidates = { cull_nodes => {}, cull_edges => {}, preserve_nodes => {}, replacement_edges => [], prune_edges => [], frontier => [], new_nodes => [], stdio_gen => { in => mk_stdio_node_generator($SRC), out => mk_stdio_node_generator($DST), }, }; + my ($flat_graph, $ops) = @_; + my ($splice_nodes, $prune_nodes); + $splice_nodes = $ops->{splice}; + $prune_nodes = $ops->{prune}; - $splice_list ||= q[]; - $prune_list ||= q[]; - my $splice_nodes = [ (split q{;}, $splice_list) ]; - my $prune_nodes = [ (split q{;}, $prune_list) ]; + my $splice_candidates = { cull_nodes => {}, cull_edges => {}, preserve_nodes => {}, replacement_edges => [], prune_edges => [], frontier => [], new_nodes => [], stdio_gen => { in => mk_stdio_node_generator($SRC), out => mk_stdio_node_generator($DST), }, }; if(@{$splice_nodes}) { $splice_candidates = register_splice_pairs($flat_graph, $splice_nodes, $SPLICE, $splice_candidates); @@ -1463,7 +1656,7 @@ sub _resolve_endpoint { my ($std_port_name, $in_out, $near_end, $far_end) = ($which_end == $SRC) ? qw/ use_STDOUT out from to / : qw/ use_STDIN in to from /; # make sure the named port actually exists on this node (by checking the edges) - if($port and not any { $_ eq $port } map { (split q/:/, $_->{$near_end})[1] } @{$node_info->{all_edges}->{$in_out}}) { croak q[port ], $port, q[ is not a port of node ], $node_id; } + if($port and not any { defined $_ and $_ eq $port } map { (split q/:/, $_->{$near_end})[1] } @{$node_info->{all_edges}->{$in_out}}) { croak q[port ], $port, q[ is not a port of node ], $node_id; } # if port name is q[], STD[IN|OUT] is implied. Is this possible for this node? if(not $port and $node_info->{node}->{type} !~ /\A(IN|OUT|RA)FILE\Z/smx and not $node_info->{node}->{$std_port_name}) { croak q[port not specified by name, but ], $std_port_name, q[ is false for node ], $node_id; } @@ -1621,25 +1814,33 @@ sub split_csl { # an empty initial param_store is added. ###################################################################### sub initialise_params { - my ($keys, $vals, $nullkeys, $param_vals_fns) = @_; + my ($keys, $vals, $nullkeys, $splice_list, $prune_list, $param_vals_fns) = @_; my $pv = {}; $pv = construct_pv($keys, $vals, $nullkeys); - return combine_pvs($param_vals_fns, $pv); + $pv = combine_pvs($param_vals_fns, $pv); + + $pv = add_ops($pv, $splice_list, $prune_list); + + return $pv; } sub construct_pv { my ($keys, $vals, $nullkeys) = @_; - my $pv; - my $subst_requests = {}; - my $subst_map_overrides = {}; + my $pv = { param_store => [], assign => [], assign_local => {}, ops => { splice => [], prune => [], }, };; + my $subst_requests; + my $subst_map_overrides; if(@$keys != @$vals) { croak q[Mismatch between keys and vals]; } + if(@{$keys} == 0 and @{$nullkeys} == 0) { + return; + } + for my $nullkey (@$nullkeys) { $subst_requests->{$nullkey} = undef; } @@ -1673,10 +1874,11 @@ sub construct_pv { $subst_requests->{$param_name} = $param_value; } } - - $pv = { param_store => [], assign => [ $subst_requests ], assign_local => $subst_map_overrides, }; } + if(defined $subst_requests) { $pv->{assign} = [ $subst_requests ]; }; + if(defined $subst_map_overrides) { $pv->{assign_local} = $subst_map_overrides; }; + return $pv; } @@ -1690,7 +1892,15 @@ sub construct_pv { ################################################################################### sub combine_pvs { my ($param_vals_fns, $clpv) = @_; - my $target = {}; + my $target = { + assign => [], + assign_local => {}, + param_store => [], + ops => { + splice => [], + prune => [], + } + }; my @all_pvs = (); # read the pv data from files, add to list @@ -1734,14 +1944,30 @@ sub combine_pvs { $target->{assign} = [ merge($target->{assign}->[0], $pv->{assign}->[0]) ]; $target->{assign_local} = merge($target->{assign_local}, $pv->{assign_local}); + + if($pv->{ops}->{splice} and ref $pv->{ops}->{splice} eq q[ARRAY]) { push @{$target->{ops}->{splice}}, @{$pv->{ops}->{splice}}; } + if($pv->{ops}->{prune} and ref $pv->{ops}->{prune} eq q[ARRAY]) { push @{$target->{ops}->{prune}}, @{$pv->{ops}->{prune}}; } } - $target->{assign} ||= []; - $target->{assign_local} ||= {}; - $target->{param_store} ||= []; return $target; } +###################################################################### +# add_ops: +# lightly parse any splice or prune from the command line and add them +# to the ops section of the param_vals hash +###################################################################### +sub add_ops { + my ($pv, $splice_list, $prune_list) = @_; + + $pv->{ops}->{splice} ||= []; + $pv->{ops}->{prune} ||= []; + if($splice_list) { push @{$pv->{ops}->{splice}}, ( (split q{;}, $splice_list) ) }; + if($prune_list) { push @{$pv->{ops}->{prune}}, ( (split q{;}, $prune_list) ) }; + + return $pv; +} + ######################################################### # _parse_localised_param_name # this should allow for escaping of the delimiter (TODO) @@ -1976,7 +2202,7 @@ sub mkewi { for my $ewi_item (@list) { if($ewi_item->{type} == $EWI_ERROR and $ewi_item->{subclass} <= $fatality_level) { - my $tag = ${$ewi_item->{tag}}; + my $tag = (defined $ewi_item->{tag} and ref $ewi_item->{tag} eq q[SCALAR] and ${$ewi_item->{tag}}); if($exclude_list and $tag and any { /$tag/ } @{$exclude_list}) { return 0; # disregard this "error" } diff --git a/t/10-vtfp-pv.t b/t/10-vtfp-pv.t index b4d593d27..34a646211 100644 --- a/t/10-vtfp-pv.t +++ b/t/10-vtfp-pv.t @@ -1,12 +1,16 @@ use strict; use warnings; use Carp; -use Test::More tests => 3; +use Test::More tests => 4; +use File::Slurp; use Perl6::Slurp; use JSON; use File::Temp qw(tempdir); +use Data::Dumper; + my $tdir = tempdir(CLEANUP => 1); + my $template = q[t/data/10-vtfp-pv.json]; my $pv_file = $tdir.q[/10-vtfp-pv.pv]; my $processed_template = $tdir.q[/10-vtfp-pv-processed.json]; @@ -17,7 +21,7 @@ subtest 'pv0' => sub { system(qq[bin/vtfp.pl -verbosity_level 0 -o $processed_template -export_param_vals $pv_file $template]) == 0 or croak q[Failed to export params]; my $pv_data = from_json(slurp $pv_file); - my $expected = {assign_local => {} ,param_store => [], assign => []}; + my $expected = {assign_local => {} ,param_store => [], assign => [], ops => { splice => [], prune => [], }}; is_deeply ($pv_data, $expected, '(ts1) exported parameter values as expected'); my $vtfp_results = from_json(slurp "bin/vtfp.pl -verbosity_level 0 -param_vals $pv_file $template |"); @@ -33,7 +37,7 @@ subtest 'pv1' => sub { system(qq[bin/vtfp.pl -verbosity_level 0 -o $processed_template -export_param_vals $pv_file -keys subject,adj -vals party,deafening $template]) == 0 or croak q[Failed to export params]; my $pv_data = from_json(slurp $pv_file); - my $expected = {assign_local => {} ,param_store => [], assign => [ {subject => q~party~, adj =>q~deafening~, }]}; + my $expected = {assign_local => {} ,param_store => [], assign => [ {subject => q~party~, adj =>q~deafening~, }], ops => { splice => [], prune => [], }}; is_deeply ($pv_data, $expected, '(ts2) exported parameter values as expected'); my $vtfp_results = from_json(slurp "bin/vtfp.pl -verbosity_level 0 -param_vals $pv_file $template |"); @@ -49,7 +53,7 @@ subtest 'pv2' => sub { system(qq[bin/vtfp.pl -verbosity_level 0 -o $processed_template -export_param_vals $pv_file -keys subject,prepobj -vals world,whimper -nullkeys adj $template]) == 0 or croak q[Failed to export params]; my $pv_data = from_json(slurp $pv_file); - my $expected = {assign_local => {} ,param_store => [], assign => [ {subject => q~world~, prepobj => q~whimper~, adj => undef}]}; + my $expected = {assign_local => {} ,param_store => [], assign => [ {subject => q~world~, prepobj => q~whimper~, adj => undef}], ops => { splice => [], prune => [], }}; is_deeply ($pv_data, $expected, '(ts3) exported parameter values as expected'); my $vtfp_results = from_json(slurp "bin/vtfp.pl -verbosity_level 0 -param_vals $pv_file $template |"); @@ -59,4 +63,147 @@ subtest 'pv2' => sub { is_deeply ($vtfp_results, $expected, '(ts4) json config generated using pv file as expected'); }; +# test node splice/prune via param file +subtest 'pv3' => sub { + plan tests => 2; + + my $basic_linear_template = { + description => q[simple linear chain (stdin->stdout) of nodes], + version => q[1.0], + nodes => + [ + { + id => q[hello], + type => q[EXEC], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ q/echo/, q/Hello/ ], + }, + { + id => q[rev], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/rev/ ] + }, + { + id => q[uc], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/tr/, q/[:lower:]/, q/[:upper:]/ ], + }, + { + id => q[disemvowel], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/tr/, q/-d/, q/[aeiouAEIOU]/ ], + }, + { + id => q[output], + type => q[OUTFILE], + name => q/tmp.xxx/, + }, + ], + edges => + [ + { id => q[e0], from => q[hello], to => q[rev] }, + { id => q[e1], from => q[rev], to => q[uc] }, + { id => q[e2], from => q[uc], to => q[disemvowel] }, + { id => q[e3], from => q[disemvowel], to => q[output] } + ] + }; + + my $template = $tdir.q[/10-vtfp-pv_splice.json]; + my $template_contents = to_json($basic_linear_template); + write_file($template, $template_contents); + + my $splice_pv = {assign_local => {} ,param_store => [], assign => [ ], ops => { splice => [ q{rev}, ], prune => [], }}; + my $pv = $tdir.q[/10-vtfp-pv_splice_prune_pvin.json]; + my $pv_contents = to_json($splice_pv); + write_file($pv, $pv_contents); + + my $vtfp_results = from_json(slurp "bin/vtfp.pl -verbosity_level 0 -no-absolute_program_paths -param_vals $pv $template |"); + + my $expected = { nodes => + [ + { + id => q[hello], + type => q[EXEC], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ q/echo/, q/Hello/ ], + }, + { + id => q[uc], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/tr/, q/[:lower:]/, q/[:upper:]/ ], + }, + { + id => q[disemvowel], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/tr/, q/-d/, q/[aeiouAEIOU]/ ], + }, + { + id => q[output], + type => q[OUTFILE], + name => q/tmp.xxx/, + }, + ], + edges => + [ + { id => q/e2/, from => q/uc/, to => q/disemvowel/ }, + { id => q/e3/, from => q/disemvowel/, to => q/output/ }, + { id => q/hello_to_uc/, from => q/hello/, to => q/uc/ }, + ] + }; + + is_deeply ($vtfp_results, $expected, '(pv3) one node in a chain spliced out'); + + my $prune_pv = {assign_local => {} ,param_store => [], assign => [ ], ops => { splice => [], prune => [ q{disemvowel-}], }}; + $pv = $tdir.q[/10-vtfp-pv_prune_pvin.json]; + $pv_contents = to_json($prune_pv); + write_file($pv, $pv_contents); + + $vtfp_results = from_json(slurp "bin/vtfp.pl -verbosity_level 0 -no-absolute_program_paths -param_vals $pv $template |"); + + $expected = { + nodes => + [ + { + id => q[hello], + type => q[EXEC], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ q/echo/, q/Hello/ ], + }, + { + id => q[rev], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/rev/ ], + }, + { + id => q[uc], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::false, + cmd => [ q/tr/, q/[:lower:]/, q/[:upper:]/ ], + }, + ], + edges => + [ + { id => q/e0/, from => q/hello/, to => q/rev/ }, + { id => q/e1/, from => q/rev/, to => q/uc/ }, + ] + }; + is_deeply ($vtfp_results, $expected, '(pv3) prune final two nodes in a chain (output to STDOUT switched off)'); +}; + 1; diff --git a/t/10-vtfp-select_directive.t b/t/10-vtfp-select_directive.t new file mode 100644 index 000000000..10d07c01c --- /dev/null +++ b/t/10-vtfp-select_directive.t @@ -0,0 +1,884 @@ +use strict; +use warnings; +use Carp; +use Test::More tests => 6; +use Test::Cmd; +use File::Slurp; +use Perl6::Slurp; +use JSON; +use File::Temp qw(tempdir); +use Cwd; + +my $tdir = tempdir(CLEANUP => 1); + +my $odir = getcwd(); +my $test = Test::Cmd->new( prog => $odir.'/bin/vtfp.pl', workdir => q()); +ok($test, 'made test object'); + +# simple test of select directive (cases array) +subtest 'select_directive_array' => sub { + plan tests => 8; + + my $select_cmd_template = { + description => 'Test select option, allowing default them explicitly setting the select value', + version => '1.0', + subst_params => [ + { id => 'happy', default => 0 }, + ], + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => { + select => "happy", + cases => + [ + [ 'echo', "bleh" ], + [ 'echo', "woohoo" ], + ], + } + } + ] + }; + + my $template = $tdir.q[/10-vtfp-select_cmd_0_0.json]; + my $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo','bleh',], + } + ], + edges=> [], + }; + + is_deeply ($vtfp_results, $expected_result, 'allow select value to default'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys happy -vals 1 -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo','woohoo',], + } + ], + edges=> [], + }; + + is_deeply ($vtfp_results, $expected_result, 'set select value to non-default'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys happy -vals 0 -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo','bleh',], + } + ], + edges=> [], + }; + + is_deeply ($vtfp_results, $expected_result, 'explicit setting of select value to default'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys happy -vals 16 -verbosity_level 0 $template]); + cmp_ok($exit_status>>8, q(==), 255, "expected exit status of 255 for select with cases array - index out of bounds"); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys happy -vals '-1' -verbosity_level 0 $template]); + cmp_ok($exit_status>>8, q(==), 255, "expected exit status of 255 for select with cases array - negative index"); +}; + +# simple test of select directive (cases hash) +subtest 'select_directive_hash' => sub { + plan tests => 9; + + my $select_cmd_template = { + description => 'Test select option, allowing default them explicitly setting the select value', + version => '1.0', + subst_params => [ + { id => 'mood', default => 'indifferent' }, + ], + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => { + select => "mood", + cases => + { + sad => [ 'echo', "whinge" ], + happy => [ 'echo', "woohoo" ], + indifferent => [ 'echo', "meh" ] + }, + } + } + ] + }; + + my $template = $tdir.q[/10-vtfp-select_cmd_1_0.json]; + my $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo','meh',], + } + ], + edges=> [], + }; + + is_deeply ($vtfp_results, $expected_result, 'allow select value to default'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys mood -vals happy -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo','woohoo',], + } + ], + edges=> [], + }; + + is_deeply ($vtfp_results, $expected_result, 'set select value to non-default'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys mood -vals indifferent -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo','meh',], + } + ], + edges=> [], + }; + + is_deeply ($vtfp_results, $expected_result, 'explicit setting of select value to default'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys mood -vals sad -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo','whinge',], + } + ], + edges=> [], + }; + + is_deeply ($vtfp_results, $expected_result, 'another non-default setting of select value'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys mood -vals ecstatic -verbosity_level 0 $template]); + cmp_ok($exit_status>>8, q(==), 255, "expected exit status of 255 for select with cases hash - index key not present in cases"); +}; + +# single switch change batch of values +subtest 'select_directive_single_switch_batch_values' => sub { + plan tests => 8; + + my $select_cmd_template = { + version => "1.0", + description => "batch of values assigned using one select value", + subst_params => [ + { + select => "mslang", default => "eng", + cases => { + eng => [ + { id => "valA", default => "one" }, + { id => "valB", default => "two" }, + { id => "valC", default => "three" } + ], + deu => [ + { id => "valA", default => "eins" }, + { id => "valB", default => "zwei" }, + { id => "valC", default => "drei" } + ], + spa => [ + { id => "valA", default => "uno" }, + { id => "valB", default => "dos" }, + { id => "valC", default => "tres" } + ] + } + } + ], + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {subst => "valA", ifnull => "one"} ] + }, + { + id => "B", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {subst => "valB", ifnull => "two"} ] + }, + { + id => "C", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {subst => "valC", ifnull => "three"} ] + }, + { + id => "P", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => ["paste", "__A_IN__", "__B_IN__", "__C_IN__" ] + } + ], + edges => [ + { id => "AP", from => "A", to => "P:__A_IN__" }, + { id => "BP", from => "B", to => "P:__B_IN__" }, + { id => "CP", from => "C", to => "P:__C_IN__" } + ] + }; + + my $template = $tdir.q[/10-vtfp-select_cmd_2_0.json]; + my $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "one" ], + use_STDIN => JSON::false, + id => "A" + }, + { + use_STDIN => JSON::false, + cmd => [ "echo", "two" ], + id => "B", + type => "EXEC", + use_STDOUT => JSON::true + }, + { + id => "C", + cmd => [ "echo", "three" ], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + type => "EXEC" + }, + { + id => "P", + use_STDIN => JSON::false, + cmd => [ "paste", "__A_IN__", "__B_IN__", "__C_IN__" ], + use_STDOUT => JSON::true, + type => "EXEC" + } + ], + edges => [ + { from => "A", to => "P:__A_IN__", id => "AP" }, + { from => "B", id => "BP", to => "P:__B_IN__" }, + { from => "C", id => "CP", to => "P:__C_IN__" } + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'single switch/batch of values with default'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys mslang -vals deu -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "eins" ], + use_STDIN => JSON::false, + id => "A" + }, + { + use_STDIN => JSON::false, + cmd => [ "echo", "zwei" ], + id => "B", + type => "EXEC", + use_STDOUT => JSON::true + }, + { + id => "C", + cmd => [ "echo", "drei" ], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + type => "EXEC" + }, + { + id => "P", + use_STDIN => JSON::false, + cmd => [ "paste", "__A_IN__", "__B_IN__", "__C_IN__" ], + use_STDOUT => JSON::true, + type => "EXEC" + } + ], + edges => [ + { from => "A", to => "P:__A_IN__", id => "AP" }, + { from => "B", id => "BP", to => "P:__B_IN__" }, + { from => "C", id => "CP", to => "P:__C_IN__" } + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'single switch changes batch of values, select deu'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys mslang -vals spa -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "uno" ], + use_STDIN => JSON::false, + id => "A" + }, + { + use_STDIN => JSON::false, + cmd => [ "echo", "dos" ], + id => "B", + type => "EXEC", + use_STDOUT => JSON::true + }, + { + id => "C", + cmd => [ "echo", "tres" ], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + type => "EXEC" + }, + { + id => "P", + use_STDIN => JSON::false, + cmd => [ "paste", "__A_IN__", "__B_IN__", "__C_IN__" ], + use_STDOUT => JSON::true, + type => "EXEC" + } + ], + edges => [ + { from => "A", to => "P:__A_IN__", id => "AP" }, + { from => "B", id => "BP", to => "P:__B_IN__" }, + { from => "C", id => "CP", to => "P:__C_IN__" } + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'single switch/batch of values, select spa'); + + # same basic template as above, but without a default value for the select + $select_cmd_template = { + version => "1.0", + description => "batch of values assigned using one select value (no default)", + subst_params => [ + { + select => "mslang", + cases => { + eng => [ + { id => "valA", default => "one" }, + { id => "valB", default => "two" }, + { id => "valC", default => "three" } + ], + deu => [ + { id => "valA", default => "eins" }, + { id => "valB", default => "zwei" }, + { id => "valC", default => "drei" } + ], + spa => [ + { id => "valA", default => "uno" }, + { id => "valB", default => "dos" }, + { id => "valC", default => "tres" } + ] + } + } + ], + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {subst => "valA", ifnull => "unspec"} ] + }, + { + id => "B", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {subst => "valB", ifnull => "unspec"} ] + }, + { + id => "C", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {subst => "valC", ifnull => "unspec"} ] + }, + { + id => "P", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => ["paste", "__A_IN__", "__B_IN__", "__C_IN__" ] + } + ], + edges => [ + { id => "AP", from => "A", to => "P:__A_IN__" }, + { id => "BP", from => "B", to => "P:__B_IN__" }, + { id => "CP", from => "C", to => "P:__C_IN__" } + ] + }; + + $template = $tdir.q[/10-vtfp-select_cmd_2_1.json]; + $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "unspec" ], + use_STDIN => JSON::false, + id => "A" + }, + { + use_STDIN => JSON::false, + cmd => [ "echo", "unspec" ], + id => "B", + type => "EXEC", + use_STDOUT => JSON::true + }, + { + id => "C", + cmd => [ "echo", "unspec" ], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + type => "EXEC" + }, + { + id => "P", + use_STDIN => JSON::false, + cmd => [ "paste", "__A_IN__", "__B_IN__", "__C_IN__" ], + use_STDOUT => JSON::true, + type => "EXEC" + } + ], + edges => [ + { from => "A", to => "P:__A_IN__", id => "AP" }, + { from => "B", id => "BP", to => "P:__B_IN__" }, + { from => "C", id => "CP", to => "P:__C_IN__" } + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'single switch/batch of values without default'); +}; + +# multisel +subtest 'select_directive_multi_sel' => sub { + plan tests => 16; + + my $select_cmd_template = { + version => "1.0", + description => "select multiple values from cases (array)", + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {select => "wordsel", cases => [ "one", "two", "three", "four" ]} ] + }, + ], + }; + + my $template = $tdir.q[/10-vtfp-select_cmd_3_0.json]; + my $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys wordsel,wordsel -vals 1,3 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "two", "four" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select multiple values from cases (array)'); + + $select_cmd_template = { + version => "1.0", + description => "select multiple values from cases (hash)", + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {select => "wordsel", cases => { first => "one", second => "two", third => "three", fourth => "four" }} ] + }, + ], + }; + + $template = $tdir.q[/10-vtfp-select_cmd_3_1.json]; + $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys wordsel,wordsel -vals second,fourth $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "two", "four" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select multiple values from cases (hash)'); + + $select_cmd_template = { + version => "1.0", + description => "select multiple values from cases (array)", + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {select => "wordsel", "select_range" => [2,3], cases => [ "one", "two", "three", "four" ]} ] + }, + ], + }; + + $template = $tdir.q[/10-vtfp-select_cmd_3_2.json]; + $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys wordsel -vals 1 $template]); + cmp_ok($exit_status>>8, q(==), 255, "expected exit status of 255 for select with select range - too few index keys (only 1, must be between 2 and 3)"); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys wordsel,wordsel -vals 1,2 $template]); + cmp_ok($exit_status>>8, q(==), 0, "expected exit status of 0 for select with select range - acceptable number of index keys (2, must be between 2 and 3)"); + $vtfp_results = from_json($test->stdout); + $expected_result = { + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "two", "three" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'valid select with select_range (2 with 2-3)'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys wordsel,wordsel,wordsel -vals 0,1,2 $template]); + cmp_ok($exit_status>>8, q(==), 0, "expected exit status of 0 for select with select range - acceptable number of index keys (3, must be between 2 and 3)"); + $vtfp_results = from_json($test->stdout); + $expected_result = { + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "one", "two", "three" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'valid select with select_range (3 with 2-3)'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys wordsel,wordsel,wordsel,wordsel -vals 0,1,2,3 $template]); + cmp_ok($exit_status>>8, q(==), 255, "expected exit status of 255 for select with select range - too few index keys (4, must be between 2 and 3)"); + + $select_cmd_template = { + version => "1.0", + description => "select multiple non-unique values from cases (array)", + subst_params => [ + { + id => "notes", + default => { + select => "tune", + cases => { + happybirthday => [ 4, 4, 5, 4, 0, 6, 4, 4, 5, 4, 1, 0, 4, 4, 4, 2, 1, 0, 6, 2, 2, 1, 0, 1, 0 ], + scale => [ 0, 1, 2, 3, 4, 5, 6 ] + } + } + } + ], + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", + { select => "notes", + cases => [ "do", "re", "mi", "fa", "so", "la", "ti" ], + default => [ 0, 2, 2, 2, 4, 4, 1, 3, 3, 5, 6, 6 ] + } + ], + }, + ], + }; + + $template = $tdir.q[/10-vtfp-select_cmd_3_3.json]; + $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "do", "mi", "mi", "mi", "so", "so", "re", "fa", "fa", "la", "ti", "ti" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select multiple non-unique values from cases (array)'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys tune -vals happybirthday $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "so", "so", "la", "so", "do", "ti", "so", "so", "la", "so", "re", "do", "so", "so", "so", "mi", "re", "do", "ti", "mi", "mi", "re", "do", "re", "do" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select multiple non-unique values from cases (array) (hb)'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys tune -vals scale $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "do", "re", "mi", "fa", "so", "la", "ti" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select multiple non-unique values from cases (array) (scale)'); +}; + +### no_sel - explicitly switch off +subtest 'select_directive_no_sel' => sub { + plan tests => 8; + + # with array cases + my $select_cmd_template = { + version => "1.0", + description => "select multiple values from cases (array)", + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {select => "wordsel", "default" => 3, cases => [ "one", "two", "three", "four" ]} ] + }, + ], + }; + + my $template = $tdir.q[/10-vtfp-select_cmd_4_0.json]; + my $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "four" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select value using default from cases (array)'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys fred -vals bloggs -nullkeys wordsel $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select no values from cases (array), overriding default with nullkeys'); + + # now the same but with hash cases + $select_cmd_template = { + version => "1.0", + description => "select multiple values from cases (array)", + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {select => "wordsel", "default" => "fourth", cases => { first => "one", second => "two", third => "three", fourth => "four" }} ] + }, + ], + }; + + $template = $tdir.q[/10-vtfp-select_cmd_4_1.json]; + $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "four" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select value using default from cases (hash)'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys fred -vals bloggs -nullkeys wordsel $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select no values from cases (hash), overriding default with nullkeys'); +}; + +1; diff --git a/t/10-vtfp-splice_nodes.t b/t/10-vtfp-splice_nodes.t index 19898ab86..1d6331848 100644 --- a/t/10-vtfp-splice_nodes.t +++ b/t/10-vtfp-splice_nodes.t @@ -10,7 +10,6 @@ use File::Temp qw(tempdir); use Cwd; my $tdir = tempdir(CLEANUP => 1); -print q[tdir: ], $tdir, "\n"; my $basic_linear_template = { description => q[simple linear chain (stdin->stdout) of nodes], @@ -783,6 +782,7 @@ subtest 'spl1' => sub { is_deeply ($vtfp_results, $expected, '(spl1) prune ends of two branches using wildcard'); }; + # exec failures subtest 'spl2' => sub { plan tests => 2; @@ -798,4 +798,5 @@ subtest 'spl2' => sub { cmp_ok($exit_status>>8, q(==), 255, "expected exit status of 255 for splice fail test. Two sequential nodes (without port spec). An error because the first splice implies preservation of the second node to be splice out"); }; + 1; diff --git a/t/10-vtfp-subst_directive.t b/t/10-vtfp-subst_directive.t index 4b276fb06..a00f4ed43 100644 --- a/t/10-vtfp-subst_directive.t +++ b/t/10-vtfp-subst_directive.t @@ -39,7 +39,7 @@ subtest 'flat_array_param_names' => sub { ] }; - my $template = $tdir.q[/10-vtfp-lat_array_param_names.json]; + my $template = $tdir.q[/10-vtfp-select_0_0.json]; my $template_contents = to_json($flat_array_param_names_template); write_file($template, $template_contents); @@ -93,7 +93,7 @@ subtest 'indirect_param_names' => sub { ], }; - my $template = $tdir.q[/10-vtfp-indirect_param_names.json]; + my $template = $tdir.q[/10-vtfp-select_1_0.json]; my $template_contents = to_json($indirect_param_names_template); write_file($template, $template_contents); @@ -183,7 +183,7 @@ subtest 'full_node_subst' => sub { ], }; - my $template = $tdir.q[/10-vtfp-full_nodes_as_subst_template.json]; + my $template = $tdir.q[/10-vtfp-select_2_0.json]; my $template_contents = to_json($full_nodes_as_subst_template); write_file($template, $template_contents); From b226de0b7616461d896ff480689f6faef9d30e5e Mon Sep 17 00:00:00 2001 From: Kevin Lewis Date: Wed, 2 Aug 2017 14:24:01 +0100 Subject: [PATCH 02/20] Port spec v2 (#187) * updates to executables to use new port specification format * improve error messages * update and extend tests to cover new port specification format * add explicit use_STDIN and use_STDOUT attributes where needed. amend file type to RAFILE to reflect that it in an input and an output * amend test to recognise updated error message * further additions of explicit use_STDIN and use_STDOUT attributes where needed and changing file types to RAFILE to reflect that it in an input and an output --- bin/viv.pl | 337 +++++-- bin/vtfp.pl | 155 ++- data/vtlib/bamindexdecoder.json | 2 +- ...2bam_phix_deplex_wtsi_stage1_template.json | 2 +- data/vtlib/bwa_aln_alignment.json | 10 + data/vtlib/bwa_mem_alignment.json | 6 + data/vtlib/final_output_prep.json | 2 +- data/vtlib/tophat2_alignment.json | 4 +- t/10-vtfp-array_expansion.t | 11 +- t/10-vtfp-noop.t | 5 +- t/10-vtfp-param_ring.t | 12 +- t/10-vtfp-pv.t | 13 +- t/10-vtfp-select_directive.t | 22 + t/10-vtfp-select_directive.v2.t | 906 ++++++++++++++++++ t/10-vtfp-splice_nodes.t | 621 +++++++++++- t/10-vtfp-subst_directive.t | 10 + t/10-vtfp-subst_directive.v2.t | 535 +++++++++++ t/10-vtfp-vtfile.mod.t | 412 ++++++++ t/10-vtfp-vtfile.t | 8 +- t/10-vtfp-vtfile_v2.t | 412 ++++++++ t/50-viv.t | 18 +- t/data/10-vtfp-array_expansion.v2.json | 19 + t/data/50-viv_failing_io_pipeline0.v2.vtf | 41 + t/data/50-viv_failing_io_pipeline0.vtf | 1 + t/data/50-viv_failing_io_pipeline1.v2.vtf | 41 + t/data/50-viv_failing_io_pipeline1.vtf | 1 + t/data/50-viv_failing_io_pipeline2.v2.vtf | 41 + t/data/50-viv_failing_io_pipeline2.vtf | 1 + t/data/50-viv_failing_pipeline.v2.vtf | 38 + t/data/50-viv_failing_pipeline.vtf | 5 + t/data/50-viv_pipeline.v2.vtf | 41 + t/data/50-viv_pipeline.vtf | 3 +- 32 files changed, 3633 insertions(+), 102 deletions(-) create mode 100644 t/10-vtfp-select_directive.v2.t create mode 100644 t/10-vtfp-subst_directive.v2.t create mode 100644 t/10-vtfp-vtfile.mod.t create mode 100644 t/10-vtfp-vtfile_v2.t create mode 100644 t/data/10-vtfp-array_expansion.v2.json create mode 100644 t/data/50-viv_failing_io_pipeline0.v2.vtf create mode 100644 t/data/50-viv_failing_io_pipeline1.v2.vtf create mode 100644 t/data/50-viv_failing_io_pipeline2.v2.vtf create mode 100644 t/data/50-viv_failing_pipeline.v2.vtf create mode 100644 t/data/50-viv_pipeline.v2.vtf diff --git a/bin/viv.pl b/bin/viv.pl index db40cc17a..23f24569a 100755 --- a/bin/viv.pl +++ b/bin/viv.pl @@ -25,6 +25,8 @@ Readonly::Scalar my $VLMED => 2; Readonly::Scalar my $VLMAX => 3; +my $TEMPLATE_VERSION = 1; + my %opts; getopts('xshv:o:r:t:', \%opts); @@ -49,6 +51,8 @@ my $cfg = from_json($s); +if($cfg->{version}) { $TEMPLATE_VERSION = $cfg->{version}; } + ############################################### # insert any tees requested into the main graph ############################################### @@ -98,37 +102,78 @@ $logger->($VLMAX, "\n==================================\nEXEC nodes(post RAFILE processing):\n==================================\n", Dumper(%exec_nodes), "\n"); -# For each edge: -# If both "from" and "to" nodes are of type EXEC, data transfer will be done via a named pipe, -# otherwise via file whose name is determined by the non-EXEC node's name attribute (communication -# between two non-EXEC nodes is of questionable value and is currently considered an error). -for my $edge (@{$edges}) { - my ($from_node, $from_id, $from_port) = _get_node_info($edge->{from}, \%all_nodes); - my ($to_node, $to_id, $to_port) = _get_node_info($edge->{to}, \%all_nodes); - my $data_xfer_name; +my %port_list; +####################################################### +# examine all the EXEC nodes for I/O ports and add them +# to a port list indexed by node name, port name +####################################################### +for my $node (values %exec_nodes) { + $port_list{$node->{id}} = _extract_ports($node->{cmd}, $node->{id}, $node->{tver}); + if($node->{use_STDIN}) { $port_list{$node->{id}}->{__stdin__} = {attribs => { direction => q[in], type => q[stream] } }; } + if($node->{use_STDOUT}) { $port_list{$node->{id}}->{__stdout__} = {attribs => { direction => q[out], type => q[stream] } }; } +} +for my $node (values %outfile_nodes, values %rafile_nodes) { + $port_list{$node->{id}}->{__stdin__} = {attribs => { direction => q[in], type => q[stream] } }; +} +for my $node (values %infile_nodes, values %rafile_nodes) { + $port_list{$node->{id}}->{__stdout__} = {attribs => { direction => q[out], type => q[stream] } }; +} - if($from_node->{type} eq q[EXEC]) { - if($to_node->{type} eq q[EXEC]) { - $data_xfer_name = _create_fifo($edge->{from}); - } - elsif(defined $to_node->{subtype} and $to_node->{subtype} eq q[DUMMY]) { - $data_xfer_name = q[]; - } - else { - $data_xfer_name = $to_node->{name}; - } +########################################################### +# run through the edges, generate and record data_xfer_name +# and connection type in the port list +########################################################### +my $ms_stack; +for my $edge (@{$edges}) { + $ms_stack = _process_edge($edge, \%port_list, $ms_stack); +} +if($ms_stack and @{$ms_stack}) { # if any messages in @fatal_error_stack, report them and croak + my $fatal_seen = 0; + for my $ms (@{$ms_stack}) { + carp(($ms->{severity}? q[ERROR]: q[WARNING]), q[: ], $ms->{text}); + $fatal_seen ||= $ms->{severity}; } - else { - if($to_node->{type} eq q[EXEC]) { - $data_xfer_name = $from_node->{name}; - } - else { - croak q[Edges must start or terminate in an EXEC node; from: ], $from_node->{id}, q[, to: ], $to_node->{id}; + if($fatal_seen) { croak q[Terminating after fatal errors detected in edge processing]; } +} + +###################################################################### +# go through the port list, verify connections requested by edges were +# valid (number and type), identify any unused ports, and perform +# substitution of data_xfer_names for ports in the EXEC nodes +###################################################################### +for my $node_id (keys %port_list) { + for my $port_name (keys %{$port_list{$node_id}}) { + if($port_list{$node_id}->{$port_name}) { + + my $data_xfer_name = $port_list{$node_id}->{$port_name}->{data_xfer_name}; + if(not defined $data_xfer_name or $data_xfer_name eq q[]) { next; } + if($port_name eq q[__stdin__] or $port_name eq q[__stdout__]) { + my $node_edge_std = $port_name eq q[__stdout__]? q[STDOUT]: q[STDIN]; + my $node = $exec_nodes{$node_id}; + if($node->{$node_edge_std}){ + croak "Cannot use $node_edge_std for node ".($node->{'id'}).' more than once'; + #TODO: allow multiple STDOUT with dup? + } + $node->{$node_edge_std} = $data_xfer_name; + } + else { + my $tver = $port_list{$node_id}->{$port_name}->{tver}; + $tver ||= $TEMPLATE_VERSION; + if($tver < 2) { + my $node = $port_list{$node_id}->{$port_name}->{node}; + for my $cmd_part (ref $node->{cmd} eq 'ARRAY' ? @{$node->{cmd}}: $node->{cmd}) { + $cmd_part =~ s/\Q$port_name\E/$data_xfer_name/smx; + } + my $xxx = q[nothing]; + } + else { + for my $loc (@{$port_list{$node_id}->{$port_name}->{occurrences}}) { + splice @{$loc->{arr}}, $loc->{idx}, 1, $data_xfer_name; + } + } + } } } - - _update_node_data_xfer($from_node, $from_port, $data_xfer_name, $FROM); - _update_node_data_xfer($to_node, $to_port, $data_xfer_name, $TO); } $logger->($VLMAX, "\n==================================\nEXEC nodes(post edges preprocessing):\n==================================\n", Dumper(%exec_nodes), "\n"); @@ -242,40 +287,209 @@ sub _create_fifo { return $output_name; } -sub _update_node_data_xfer { - my ($node, $port, $data_xfer_name, $edge_side) = @_; +###################################################### +# _process_edge: +# use edge to generate and record data_xfer_name and +# connection type in port_list +###################################################### +sub _process_edge { + my ($edge, $port_list, $ms_stack) = @_; + my ($from_node, $from_id, $from_port) = _get_node_info($edge->{from}, \%all_nodes); + my ($to_node, $to_id, $to_port) = _get_node_info($edge->{to}, \%all_nodes); + my $severity; - if($node->{type} eq q[EXEC] and $data_xfer_name ne q[]) { - if(defined $port) { - if(my($inout) = grep {$_} $port=~/_(IN|OUT)__\z/smx , $port=~/\A__(IN|OUT)_/smx ){ # if port has _{IN,OUT}_ {suf,pre}fix convention - #ensure port is connected to in manner suggested by naming convention - croak 'Node '.($node->{'id'})." port $port connected as ".($edge_side == $FROM?q("from"):q("to")) if (($inout eq q(OUT))^($edge_side == $FROM)); - } else { - croak 'Node '.($node->{'id'})." has poorly described port $port (no _{IN,OUT}__ {suf,pre}fix)\n"; - } - my $cmd = $node->{'cmd'}; - for my$cmd_part ( ref $cmd eq 'ARRAY' ? @{$cmd}[1..$#{$cmd}] : ($node->{'cmd'}) ){ - return if ($cmd_part =~ s/\Q$port\E/$data_xfer_name/smx); - } #if link for port has not been made (port never defined, or already substituted, in node cmd) bail out - croak 'Node '.($node->{'id'})." has no port $port"; + $from_port ||= q[__stdout__]; + $to_port ||= q[__stdin__]; + + $ms_stack ||= []; + if(not $port_list{$from_id}->{$from_port}) { + push @{$ms_stack}, {severity => 1, text => qq[from port $from_id:$from_port referenced in edge ($from_id:$from_port => $to_id:$to_port), but no corresponding port found in nodes\n]}; + } + if(not $port_list{$to_id}->{$to_port}) { + push @{$ms_stack}, {severity => 1, text => qq[to port $to_id:$to_port referenced in edge ($from_id:$from_port => $to_id:$to_port), but no corresponding port found in nodes\n]}; + } + if($port_list{$from_id}->{$from_port}->{attribs}->{direction} and $port_list{$from_id}->{$from_port}->{attribs}->{direction} eq q[in]) { + push @{$ms_stack}, {severity => 1, text => qq[from port $from_id:$from_port refers to input port in node\n]}; + } + if($port_list{$to_id}->{$to_port}->{attribs}->{direction} and $port_list{$to_id}->{$to_port}->{attribs}->{direction} eq q[out]) { + push @{$ms_stack}, {severity => 1, text => qq[to port to_id:$to_port refers to output port in node\n]}; + } + + # determine connection type, generate appropriate data transfer name + my ($data_xfer_name, $connection_type); + if($from_node->{type} eq q[EXEC]) { + if($to_node->{type} eq q[EXEC]) { + $data_xfer_name = _create_fifo($edge->{from}); + $connection_type = q[pipe]; + } + elsif(defined $to_node->{subtype} and $to_node->{subtype} eq q[DUMMY]) { + $data_xfer_name = q[]; + $connection_type = q[file]; } else { - my $node_edge_std = $edge_side == $FROM? q[STDOUT]: q[STDIN]; - if($node->{$node_edge_std}){ - croak "Cannot use $node_edge_std for node ".($node->{'id'}).' more than once'; - #TODO: allow multiple STDOUT with dup? + $data_xfer_name = $to_node->{name}; + $connection_type = q[file]; + } + } + else { + if($to_node->{type} eq q[EXEC]) { + $data_xfer_name = $from_node->{name}; + $connection_type = q[file]; + } + else { + push @{$ms_stack}, {severity => 1, text => qq[Edges must start or terminate in an EXEC node; from: $from_node->{id}, to: $to_node->{id}\n]}; + } + } + + # from port - check validity: previous edges? if so, compatible type (fifo/file)? if one, check other end of previous edge to make sure not many:many creation + if(exists $port_list{$from_id}->{$from_port}->{dst} and @{$port_list{$from_id}->{$from_port}->{dst}} > 0) { # an edge from this port already added + + $data_xfer_name = $port_list{$from_id}->{$from_port}->{data_xfer_name}; + + # confirm connection type match + if($connection_type ne $port_list{$from_id}->{$from_port}->{connection_type}) { + $severity = ($TEMPLATE_VERSION>=2 and (not $from_node->{tver} or $from_node->{tver} >=2))?1:0; + push @{$ms_stack}, {severity => $severity, text => qq[multiple edges from port have inconsistent connection types (pipe/file); from: $from_node->{id}, to: $to_node->{id}\n]}; + } + + # check for many:many creation + my $src_count = $port_list{$from_id}->{$from_port}->{src}? @{$port_list{$from_id}->{$from_port}->{src}}: 0; + my $dst_count = $port_list{$from_id}->{$from_port}->{dst}? @{$port_list{$from_id}->{$from_port}->{dst}}: 0; + if($dst_count == 1) { + my ($other_dst_id, $other_dst_port) = ($port_list{$from_id}->{$from_port}->{dst}->[0]->{id}, $port_list{$from_id}->{$from_port}->{dst}->[0]->{port}); + my $other_dst = $port_list{$other_dst_id}->{$other_dst_port}; + if(@{$other_dst->{src}} > 1) { + $severity = ($TEMPLATE_VERSION>=2 and (not $from_node->{tver} or $from_node->{tver} >=2))?1:0; + push @{$ms_stack}, {severity => $severity, text => qq[processing edge from $from_id:$from_port to $to_id:$to_port: illegal many:many creation\n]}; + } + } + } + + # to port - check validity: previous edges? if so, compatible type (fifo/file)? if one, check other end of previous edge to make sure not many:many creation + if(exists $port_list{$to_id}->{$to_port}->{src} and @{$port_list{$to_id}->{$to_port}->{src}} > 0) { # an edge to this port already added + + # confirm connection type match + if($connection_type ne $port_list{$to_id}->{$to_port}->{connection_type}) { + $severity = ($TEMPLATE_VERSION>=2 and (not $to_node->{tver} or $to_node->{tver} >=2))?1:0; + push @{$ms_stack}, {severity => $severity, text => qq[multiple edges to port have inconsistent connection types (pipe/file); from: $from_node->{id}, to: $to_node->{id}\n]}; + } + + # check for many:many creation + my $src_count = $port_list{$to_id}->{$to_port}->{src}? @{$port_list{$to_id}->{$to_port}->{src}}: 0; + my $dst_count = $port_list{$to_id}->{$to_port}->{dst}? @{$port_list{$to_id}->{$to_port}->{dst}}: 0; + if($src_count == 1) { + my ($other_src_id, $other_src_port) = ($port_list{$to_id}->{$to_port}->{src}->[0]->{id}, $port_list{$to_id}->{$to_port}->{src}->[0]->{port}); + my $other_src = $port_list{$other_src_id}->{$other_src_port}; + if(@{$other_src->{dst}} > 1) { + $severity = ($TEMPLATE_VERSION>=2 and (not $to_node->{tver} or $to_node->{tver} >=2))?1:0; + push @{$ms_stack}, {severity => $severity, text => qq[processing edge from $from_id:$from_port to $to_id:$to_port: illegal many:many creation\n]}; + } + } + } + + $port_list{$from_id}->{$from_port}->{connection_type} = $connection_type; + $port_list{$from_id}->{$from_port}->{dst} ||= []; + push @{$port_list{$from_id}->{$from_port}->{dst}}, { id => $to_id , port => $to_port}; + + $port_list{$to_id}->{$to_port}->{connection_type} = $connection_type; + $port_list{$to_id}->{$to_port}->{src} ||= []; + push @{$port_list{$to_id}->{$to_port}->{src}}, { id => $from_id , port => $from_port}; + + $port_list{$from_id}->{$from_port}->{data_xfer_name} = $data_xfer_name; + $port_list{$to_id}->{$to_port}->{data_xfer_name} = $data_xfer_name; + + return $ms_stack; +} + +sub _extract_ports { + my ($cmd, $id, $tver) = @_; + my %ports; + + $tver ||= $TEMPLATE_VERSION; + + if(not $cmd) { return; } + + if($tver < 2) { + my $r = ref $cmd; + if($r and $r ne q[ARRAY]) { + carp qq[Unsupported ref type $r for cmd of node $id\n]; + return; + } + + for my $elem (ref $cmd eq q[ARRAY]? @{$cmd}: ($cmd)) { + for my $port ($elem =~ /(__\S*_IN__)/gsmx, $elem=~/(__IN_\S*__)/gsmx) { + $ports{$port}->{tver} = 1; + $ports{$port}->{attribs}->{direction} = q[in]; + $ports{$port}->{attribs}->{type} = q[stream]; + $ports{$port}->{node} = $exec_nodes{$id}; } - if(exists $node->{"use_$node_edge_std"}){ - croak 'Node '.($node->{'id'})." configured not to use $node_edge_std" unless $node->{"use_$node_edge_std"}; + for my $port ($elem =~ /(__\S*_OUT__)/gsmx, $elem=~/(__OUT_\S*__)/gsmx) { + $ports{$port}->{tver} = 1; + $ports{$port}->{attribs}->{direction} = q[out]; + $ports{$port}->{attribs}->{type} = q[stream]; + $ports{$port}->{node} = $exec_nodes{$id}; } - $node->{$node_edge_std} = $data_xfer_name; } } else { - # do nothing + if(ref $cmd ne q[ARRAY]) { return; } + + for my $i (0..$#{$cmd}) { + my $elem = $cmd->[$i]; + if(ref $elem eq q[HASH]) { + if($elem->{port}) { + $ports{$elem->{port}}->{tver} = 2; + $ports{$elem->{port}}->{attribs} = reconcile_port_info($elem, $ports{$elem->{port}}->{attribs}); + $ports{$elem->{port}}->{occurrences} ||= []; + push @{$ports{$elem->{port}}->{occurrences}}, {arr => $cmd, idx => $i}; # note location for later substitution + } + elsif($elem->{packflag}) { + if(ref $elem->{packflag} eq q[ARRAY]) { + for my $j (0..$#{$elem->{packflag}}) { + my $pf_elem = $elem->{packflag}->[$j]; + if(ref $pf_elem eq q[HASH] and $pf_elem->{port}) { + $ports{$pf_elem->{port}}->{tver} = 2; + $ports{$pf_elem->{port}}->{attribs} = reconcile_port_info($pf_elem, $ports{$pf_elem->{port}}->{attribs}); + $ports{$pf_elem->{port}}->{occurrences} ||= []; + push @{$ports{$pf_elem->{port}}->{occurrences}}, {arr => $elem->{packflag}, idx => $j}; # note location for later substitution + } + } + } + else { + $logger->($VLMIN, "WARN: packflag with non-array value, node id: ", $id); + } + } + } + } } - return; + return \%ports; +} + +sub reconcile_port_info { + my ($port_elem, $current_attribs) = @_; + my $attribs; + + if($current_attribs) { # multiple occurrences of this port in the node + if($current_attribs->{direction} and $current_attribs->{direction} ne $port_elem->{direction}) { + $attribs->{direction} = q[bi]; + } + else { + $attribs->{direction} = $port_elem->{direction}; + } + if($current_attribs->{type} and $current_attribs->{type} ne $port_elem->{type}) { + $attribs->{type} = q[seekable]; # use the more restrictive option + } + else { + $attribs->{type} = $port_elem->{type}; + } + } + else { + $attribs->{direction} = $port_elem->{direction}; + $attribs->{type} = $port_elem->{type}; + } + + return $attribs; } sub _get_from_edges { @@ -298,9 +512,23 @@ sub _fork_off { my ($node, $do_exec) = @_; my $cmd = $node->{'cmd'}; my @cmd = ($cmd); - if ( ref $cmd eq 'ARRAY' ){ + + my $r = ref $cmd; + if($r eq q[ARRAY]) { @cmd = @{$cmd}; - $cmd = '[' . (join ',',@cmd) . ']'; + + # handle any packflag elements + @cmd = map { (ref $_ eq q[HASH] and $_->{packflag} and ref $_->{packflag} eq q[ARRAY])? join q//, @{$_->{packflag}} : $_ } @cmd; + + # if subtype is "stringify", then join elements into a space-separated string before exec (forces execution via comand shell interpreter when shell metacharacters are present) + if($node->{subtype} and $node->{subtype} eq q[STRINGIFY]) { + @cmd = (join ' ', @cmd); + } + # construct cmd for debug/error messages + $cmd = '[' . (join ' ',@cmd) . ']'; + } + elsif($r) { + croak q[command for node ], $node->{id}, q[ is an unsupported ref type: ], $r; } if(my $pid=fork) { # parent - record the child's departure @@ -317,6 +545,7 @@ sub _fork_off { select(STDERR);$|=1; print STDERR "Process $$ for cmd $cmd:\n"; print STDERR ' fileno(STDERR,'.(fileno STDERR).")\n"; + if(not $node->{use_STDIN}) { $node->{'STDIN'} ||= '/dev/null'; } if($node->{'STDIN'}) { sysopen STDIN, $node->{'STDIN'}, O_RDONLY|O_NONBLOCK or croak "Failed to reset STDIN, pid $$ with cmd: $cmd\n$!"; @@ -444,7 +673,7 @@ sub process_tee_list { carp q[Failed to create id for tee node for port ], $outport; next; } - my $tee_node = { id => $tnid, type => q[EXEC], use_STDIN => JSON::true, use_STDOUT => JSON::true, cmd => [ 'tee', $tee_stream_outport_name, ], }; + my $tee_node = { id => $tnid, type => q[EXEC], use_STDIN => JSON::true, use_STDOUT => JSON::true, tver => 2, cmd => [ 'tee', {port => $tee_stream_outport_name}, ], }; ######################### # create output file node diff --git a/bin/vtfp.pl b/bin/vtfp.pl index a68c6579a..88fb825d9 100755 --- a/bin/vtfp.pl +++ b/bin/vtfp.pl @@ -43,6 +43,7 @@ Readonly::Scalar my $DST => 1; Readonly::Scalar my $MIN_TEMPLATE_VERSION => 1; +my $TEMPLATE_VERSION_DEFAULT = 2; my $progname = (fileparse($0))[0]; my %opts; @@ -104,7 +105,9 @@ my $node_tree = process_vtnode(q[], $vtf_name, q[], $params, $globals); # recursively generate the vtnode tree -my $flat_graph = flatten_tree($node_tree); +my $tver_default = ($node_tree->{cfg}->{version} or $TEMPLATE_VERSION_DEFAULT); +my $flat_graph = flatten_tree($node_tree, $tver_default); +$flat_graph->{version} = $tver_default; my $cull_node_ids = []; # used to detect (and disregard) parameter substitution errors in nodes which have been removed if(@{$params->{ops}->{splice}} or @{$params->{ops}->{prune}}) { @@ -116,7 +119,7 @@ foreach my $node_with_cmd ( grep {$_->{'cmd'}} @{$flat_graph->{'nodes'}}) { - $node_with_cmd->{cmd} = finalise_array($node_with_cmd->{cmd}); + $node_with_cmd = finalise_cmd($node_with_cmd); if(not defined $node_with_cmd->{cmd} or (ref $node_with_cmd->{cmd} eq q[ARRAY] and @{$node_with_cmd->{cmd}} < 1)) { croak "command ", ($node_with_cmd->{id}? $node_with_cmd->{id}: q[NO ID]), " either empty or undefined"; } @@ -1017,16 +1020,16 @@ sub report_pv_ewi { # the resulting graph with one of the visualisation tools.) ####################################################################################### sub flatten_tree { - my ($tree_node, $flat_graph) = @_; + my ($tree_node, $tver_default, $flat_graph) = @_; $flat_graph ||= {}; # insert edges and nodes from current tree_node to $flat_graph - subgraph_to_flat_graph($tree_node, $flat_graph); + subgraph_to_flat_graph($tree_node, $tver_default, $flat_graph); # do the same recursively for any children for my $tn (@{$tree_node->{children}}) { - flatten_tree($tn, $flat_graph); + flatten_tree($tn, $tver_default, $flat_graph); } return $flat_graph; @@ -1037,7 +1040,7 @@ sub flatten_tree { # losing everything except nodes and edges is a possibly undesirable side-effect of this ######################################################################################### sub subgraph_to_flat_graph { - my ($tree_node, $flat_graph) = @_; + my ($tree_node, $tver_default, $flat_graph) = @_; my $vtnode_id = $tree_node->{id}; my $vt_name = $tree_node->{name}; @@ -1047,7 +1050,8 @@ sub subgraph_to_flat_graph { ################################################################################### # prefix the nodes in this subgraph with a prefix to ensure uniqueness of id values ################################################################################### - $subcfg->{nodes} = [ (map { $_->{id} = sprintf "%s%s", $tree_node->{node_prefix}, $_->{id}; $_; } @{$subcfg->{nodes}}) ]; + my $tver = ($subcfg->{version} or $tver_default); + $subcfg->{nodes} = [ (map { $_->{id} = sprintf "%s%s", $tree_node->{node_prefix}, $_->{id}; if($_->{type} eq q[EXEC] and not $_->{tver} and $tver ne $tver_default) { $_->{tver} = $tver; } $_; } @{$subcfg->{nodes}}) ]; ######################################################################## # any edges which refer to nodes in this subgraph should also be updated @@ -1644,8 +1648,15 @@ sub _resolve_endpoint { # use -1 argument for split to distinguish between "a" (node a) and "a:" (STDOUT of node a) my ($node_id, $port) = (split q/:/, $endpoint_spec, -1); - unless(not $port or ($which_end == $SRC and ($port=~/_OUT__\z/smx or $port=~/\A__OUT_/smx)) or ($which_end == $DST and ($port=~/_IN__\z/smx or $port=~/\A__IN_/smx))) { - croak q[port name ], $port, q[: naming convention incorrect for ], (($which_end == $SRC)? q[out] : q[in]), q[port]; + if($tver_default < 2) { + unless(not $port + or ($which_end == $SRC + and ($port=~/_OUT__\z/smx or $port=~/\A__OUT_/smx)) + or ($which_end == $DST + and ($port=~/_IN__\z/smx or $port=~/\A__IN_/smx)) + ) { + croak q[port name ], $port, q[: naming convention incorrect for ], (($which_end == $SRC)? q[out] : q[in]), q[port]; + } } my $node_info = get_node_info($node_id, $flat_graph); # fetch node and its in and out edges @@ -1776,11 +1787,23 @@ sub delete_port { my ($port_name, $node) = @_; if(ref $node->{cmd} eq 'ARRAY') { - # this sweeping approach should be replaced when syntax for port specification is improved - if(any { $_ =~ /$port_name/ } @{$node->{cmd}}) { - $node->{cmd} = [ (grep { $_ !~ /$port_name/ } @{$node->{cmd}}) ]; + if($tver_default >= 2 and (not exists $node->{tver} or $node->{tver} >= 2)) { + # remove any references from $node->{ports} section + if(exists $node->{ports}->{$port_name}) { delete $node->{ports}->{$port_name}; } + my $port_locs = _extract_ports($node->{cmd}, $node->{id}); + for my $occ (sort { $b->{idx} <=> $a->{idx} } @{$port_locs->{$port_name}->{occurrences}}) { + splice @{$occ->{arr}}, $occ->{idx}, 1; + } + return; } + else { + # this sweeping approach should be replaced when syntax for port specification is improved + if(any { $_ =~ /$port_name/ } @{$node->{cmd}}) { + $node->{cmd} = [ (grep { $_ !~ /$port_name/ } @{$node->{cmd}}) ]; + return; + } + } } else { return if ($node->{cmd} =~ s/\Q$port_name\E//smx); @@ -1791,6 +1814,69 @@ sub delete_port { return; } +sub _extract_ports { + my ($cmd, $id) = @_; + my %ports; + + unless($cmd and ref $cmd eq q[ARRAY]) { + return; + } + + for my $i (0..$#{$cmd}) { + my $elem = $cmd->[$i]; + if(ref $elem eq q[HASH]) { + if($elem->{port}) { + $ports{$elem->{port}}->{attribs} = reconcile_port_info($elem, $ports{$elem->{port}}->{attribs}); + $ports{$elem->{port}}->{occurrences} ||= []; + push @{$ports{$elem->{port}}->{occurrences}}, {arr => $cmd, idx => $i}; # note location for later substitution + } + elsif($elem->{packflag}) { + if(ref $elem->{packflag} eq q[ARRAY]) { + for my $j (0..$#{$elem->{packflag}}) { + my $pf_elem = $elem->{packflag}->[$j]; + if(ref $pf_elem eq q[HASH] and $pf_elem->{port}) { + $ports{$pf_elem->{port}}->{attribs} = reconcile_port_info($pf_elem, $ports{$pf_elem->{port}}->{attribs}); + $ports{$pf_elem->{port}}->{occurrences} ||= []; + push @{$ports{$pf_elem->{port}}->{occurrences}}, {arr => $elem->{packflag}, idx => $j}; # note location for later substitution + } + } + } + else { + $logger->($VLMIN, "WARN: packflag with non-array value, node id: ", $id); + } + } + } + } + + return \%ports; +} + +sub reconcile_port_info { + my ($port_elem, $current_attribs) = @_; + my $attribs; + + if($current_attribs) { # multiple occurrences of this port in the node + if($current_attribs->{direction} and $current_attribs->{direction} ne $port_elem->{direction}) { + $attribs->{direction} = q[bi]; + } + else { + $attribs->{direction} = $port_elem->{direction}; + } + if($current_attribs->{type} and $current_attribs->{type} ne $port_elem->{type}) { + $attribs->{type} = q[seekable]; # use the more restrictive option + } + else { + $attribs->{type} = $port_elem->{type}; + } + } + else { + $attribs->{direction} = $port_elem->{direction}; + $attribs->{type} = $port_elem->{type}; + } + + return $attribs; +} + ########################################################################### # split_csl: # split comma-separated string, allowing the commas to be escaped with '\' @@ -2058,6 +2144,51 @@ sub find_vtf { $logger->($VLFATAL, q[Failed to find vtf file: ], $vtf_fullname, q[ locally or on template_path: ], join q[:], @$template_path); } +######################################################## +# finalise_cmd: postprocess EXEC nodes +# flatten cmd element to array ref, transfer any port +# attributes from node to cmd level +######################################################## +sub finalise_cmd { + my ($cmd_node) = @_; + + $cmd_node->{cmd} = finalise_array($cmd_node->{cmd}); + + my $cmd = $cmd_node->{cmd}; + my $r=ref $cmd; + if(not $r) { + $cmd = [ $cmd ]; + } + elsif($r ne q[ARRAY]) { + $logger->($VLMAX, q[Cannot post-process node cmd element unless it is scalar or array ref (node id: ], $cmd_node->{id}, q[ ; type: ], $r, q[)] ); + return $cmd_node; + } + + if(not defined $cmd_node->{tver} or $cmd_node->{tver} >= 2) { + my $node_ports = $cmd_node->{ports}; + if(defined $node_ports) { + my %ports_seen; + for my $e (@{$cmd}) { + if(ref $e eq q[HASH] and $e->{port} and $node_ports->{$e->{port}}) { + # move all key/val pairs from node ports section to cmd port element + @{$e}{keys %{$node_ports->{$e->{port}}}} = @{$node_ports->{$e->{port}}}{keys %{$node_ports->{$e->{port}}}}; + $ports_seen{$e->{port}} = 1; + } + } + + for my $k (keys %{$node_ports}) { + if(not exists $ports_seen{$k} and defined $node_ports->{$k}->{required} and not $node_ports->{$k}->{required}) { # unless explicitly flagged as unrequired + $logger->($VLFATAL, q[Unused node port: ], $k, q[ ; node id: ], $cmd_node->{id}, q[ ; cmd: ], join q[ ], @{$cmd} ); + } + } + + delete $cmd_node->{ports}; + } + } + + return $cmd_node; +} + ############################################################## # finalise_array: used to postprocess arrays (e.g. cmd attribs # of EXEC nodes). Input is either: diff --git a/data/vtlib/bamindexdecoder.json b/data/vtlib/bamindexdecoder.json index 9baa912f7..a201b5913 100644 --- a/data/vtlib/bamindexdecoder.json +++ b/data/vtlib/bamindexdecoder.json @@ -70,7 +70,7 @@ }, { "id":"decoder_metrics", - "type":"OUTFILE", + "type":"RAFILE", "name":{"subst":"decoder_metrics"} }, { diff --git a/data/vtlib/bcl2bam_phix_deplex_wtsi_stage1_template.json b/data/vtlib/bcl2bam_phix_deplex_wtsi_stage1_template.json index 36f52416b..96668f2e9 100644 --- a/data/vtlib/bcl2bam_phix_deplex_wtsi_stage1_template.json +++ b/data/vtlib/bcl2bam_phix_deplex_wtsi_stage1_template.json @@ -436,7 +436,7 @@ "name":{"subst":"seqchksum","required":"yes","ifnull":{"subst_constructor":{ "vals":[ {"subst":"cfgdatadir"}, "/", {"subst":"fs1p","required":"yes", "ifnull": "final_stage1_process.json"} ], "postproc":{"op":"concat", "pad":""} }}}, "description":"subgraph containing final stage1 processing" }, - { "id":"seqchksum_file", "type":"OUTFILE", "name":{"subst":"seqchksum_file"} }, + { "id":"seqchksum_file", "type":"RAFILE", "name":{"subst":"seqchksum_file"} }, { "id":"filtered_bam", "type":"OUTFILE", diff --git a/data/vtlib/bwa_aln_alignment.json b/data/vtlib/bwa_aln_alignment.json index b07ee0bbd..1804788b5 100644 --- a/data/vtlib/bwa_aln_alignment.json +++ b/data/vtlib/bwa_aln_alignment.json @@ -25,6 +25,8 @@ { "id":"tee4", "type":"EXEC", + "use_STDIN":true, + "use_STDOUT":false, "cmd":[ "teepot", {"subst":"teepot_vflag", "ifnull":"-v"}, @@ -39,11 +41,15 @@ { "id":"bwa_aln_1", "type":"EXEC", + "use_STDIN":false, + "use_STDOUT":true, "cmd":[ {"subst":"bwa_executable"}, "aln", "-t", {"subst":"aligner_numthreads"}, "-b1", "__REFERENCE_GENOME_FASTA_IN__", "__BAM_IN__" ] }, { "id":"bwa_aln_2", "type":"EXEC", + "use_STDIN":false, + "use_STDOUT":true, "cmd":[ {"subst":"bwa_executable"}, "aln", "-t", {"subst":"aligner_numthreads"}, "-b2", "__REFERENCE_GENOME_FASTA_IN__", "__BAM_IN__" ] }, { @@ -63,11 +69,15 @@ { "id":"bwa_sampe", "type":"EXEC", + "use_STDIN":false, + "use_STDOUT":true, "cmd":[ {"subst":"bwa_executable"}, "sampe", "__REFERENCE_GENOME_FASTA_IN__", "__SAI_1_IN__", "__SAI_2_IN__", "__BAM_1_IN__", "__BAM_2_IN__" ] }, { "id":"samtobam", "type":"EXEC", + "use_STDIN":true, + "use_STDOUT":true, "cmd":[ "scramble", {"subst":"s2b_compress_level", "ifnull":"-0"}, diff --git a/data/vtlib/bwa_mem_alignment.json b/data/vtlib/bwa_mem_alignment.json index 171a0c35f..718a11adf 100644 --- a/data/vtlib/bwa_mem_alignment.json +++ b/data/vtlib/bwa_mem_alignment.json @@ -39,17 +39,23 @@ { "id":"bamtofastq", "type":"EXEC", + "use_STDIN":true, + "use_STDOUT":true, "cmd":["bamtofastq"] }, { "id":"bwa_mem", "comment":"presuming interleaved FR fastq records (-p flag), output all records (-T 0)", "type":"EXEC", + "use_STDIN":false, + "use_STDOUT":true, "cmd":[ {"subst":"bwa_executable"}, "mem", "-t", {"subst":"aligner_numthreads"}, {"subst":"bwa_mem_p_flag"},{"subst":"bwa_mem_Y_flag"}, {"subst":"bwa_mem_T_flag"}, {"subst":"bwa_mem_K_flag"}, "__DB_PREFIX_REFERENCE_GENOME_IN__", "__FQ_IN__" ] }, { "id":"samtobam", "type":"EXEC", + "use_STDIN":true, + "use_STDOUT":true, "cmd":[ "scramble", {"subst":"s2b_mt", "ifnull":{"subst_constructor":{ "vals":[ "-t", {"subst":"s2b_mt_val"} ]}}}, diff --git a/data/vtlib/final_output_prep.json b/data/vtlib/final_output_prep.json index 780bf5458..33e89e932 100644 --- a/data/vtlib/final_output_prep.json +++ b/data/vtlib/final_output_prep.json @@ -376,7 +376,7 @@ { "id":"bam_file", "type":"OUTFILE", "name":{"subst":"bam_file"} }, { "id":"cram_file", "type":"OUTFILE", "name":{"subst":"cram_file"} }, { "id":"cram_md5", "type":"OUTFILE", "name":{"subst":"cram_md5"} }, - { "id":"seqchksum_file", "type":"OUTFILE", "name":{"subst":"seqchksum_file"} }, + { "id":"seqchksum_file", "type":"RAFILE", "name":{"subst":"seqchksum_file"} }, { "id":"seqchksum_file_cram", "type":"RAFILE", "name":{"subst":"seqchksum_file_cram"}, "comment":"this file is a temporary fix for blocking problems at the cmp_seqchksum node" }, { "id":"seqchksum_extrahash_file", "type":"OUTFILE", "name":{"subst":"seqchksum_extrahash_file"} }, { "id":"stats_F0x900_file", "type":"OUTFILE", "name":{"subst":"stats_F0x900_file"} }, diff --git a/data/vtlib/tophat2_alignment.json b/data/vtlib/tophat2_alignment.json index ba4d2ce58..7add77269 100644 --- a/data/vtlib/tophat2_alignment.json +++ b/data/vtlib/tophat2_alignment.json @@ -190,7 +190,7 @@ "id":"tophat2", "type":"EXEC", "use_STDIN": false, - "use_STDOUT": false, + "use_STDOUT": true, "cmd": [ "tophat2", "--keep-fasta-order", @@ -241,6 +241,8 @@ { "id":"bamcat", "type":"EXEC", + "use_STDIN":false, + "use_STDOUT":true, "cmd":[ "bamcat", "I=__IN_BAM1__", "I=__IN_BAM2__", "level=0" ] }, { diff --git a/t/10-vtfp-array_expansion.t b/t/10-vtfp-array_expansion.t index 2acba6c14..2e4c90029 100644 --- a/t/10-vtfp-array_expansion.t +++ b/t/10-vtfp-array_expansion.t @@ -1,16 +1,21 @@ use strict; use warnings; -use Test::More tests => 1; +use Test::More tests => 2; use Perl6::Slurp; use JSON; my $template = q[t/data/10-vtfp-array_expansion.json]; +my $template_v2 = q[t/data/10-vtfp-array_expansion.v2.json]; { # the template contains a set of possibly multivalued parameters p1 - p5 which should expand into a list (array ref) in the cmd attribute of its one node -my $vtfp_results = from_json(slurp "bin/vtfp.pl $template |"); -my $c = {edges=> [], nodes => [ {cmd => [q~/bin/echo~,q~1A~,q~1B~,q~2A~,q~3A~,q~3B~,q~3C~,q~3D~,q~4A~,q~5A~,q~5B~], type => q~EXEC~, id => q~n1~}]}; +my $vtfp_results = from_json(slurp "bin/vtfp.pl -verbosity_level 0 $template |"); +my $c = {version => q[1.0], edges=> [], nodes => [ {cmd => [q~/bin/echo~,q~1A~,q~1B~,q~2A~,q~3A~,q~3B~,q~3C~,q~3D~,q~4A~,q~5A~,q~5B~], type => q~EXEC~, id => q~n1~, }]}; is_deeply ($vtfp_results, $c, 'first array element expansion test'); + +$vtfp_results = from_json(slurp "bin/vtfp.pl -verbosity_level 0 $template_v2 |"); +$c = {version => q[2.0], edges=> [], nodes => [ {cmd => [q~/bin/echo~,q~1A~,q~1B~,q~2A~,q~3A~,q~3B~,q~3C~,q~3D~,q~4A~,q~5A~,q~5B~], type => q~EXEC~, id => q~n1~, }]}; +is_deeply ($vtfp_results, $c, 'first array element expansion test (v2)'); } diff --git a/t/10-vtfp-noop.t b/t/10-vtfp-noop.t index 6f707439e..93a22a18d 100644 --- a/t/10-vtfp-noop.t +++ b/t/10-vtfp-noop.t @@ -11,9 +11,10 @@ my $template = q[t/data/10-vtfp-noop.json]; # the template contains nodes and edges with no parameters. This test should confirm that vtfp.pl has no effect on them. my $fc = from_json(read_file($template)); -my $expected->{nodes} = $fc->{nodes}; +my $expected->{version} = q/1.0/; +$expected->{nodes} = $fc->{nodes}; $expected->{edges} = $fc->{edges}; -my $vtfp_results = from_json(slurp "bin/vtfp.pl --no-absolute_program_paths $template |"); +my $vtfp_results = from_json(slurp "bin/vtfp.pl -verbosity_level 0 --no-absolute_program_paths $template |"); is_deeply ($vtfp_results, $expected, 'noop test'); } diff --git a/t/10-vtfp-param_ring.t b/t/10-vtfp-param_ring.t index b633d63c9..5b1fc3aa7 100644 --- a/t/10-vtfp-param_ring.t +++ b/t/10-vtfp-param_ring.t @@ -29,8 +29,8 @@ subtest 'irdetect' => sub { # Remove infinite recursion error by giving one of the parameters a value (I arbitrarily selected p3). Parameters # p1 - p4 will appear in the resulting cmd attribute (if they are set). -my $vtfp_results = from_json(slurp "bin/vtfp.pl -keys p3 -vals break $template |"); -my $c = {edges=> [], nodes => [ {cmd => [q~/bin/echo~,q~one~,q~break~,q~break~,q~break~], type => q~EXEC~, id => q~n1~}]}; +my $vtfp_results = from_json(slurp "bin/vtfp.pl -verbosity_level 0 -keys p3 -vals break $template |"); +my $c = {version => q[1.0], edges=> [], nodes => [ {cmd => [q~/bin/echo~,q~one~,q~break~,q~break~,q~break~], type => q~EXEC~, id => q~n1~}]}; is_deeply ($vtfp_results, $c, 'first parameter ring test'); } @@ -38,8 +38,8 @@ is_deeply ($vtfp_results, $c, 'first parameter ring test'); # this is a slightly more complicated version of the previous test. It sets two parameter values in the ring, and one # not involved the ring (p1) -my $vtfp_results = from_json(slurp "bin/vtfp.pl -keys p1,p2,p4 -vals first,second,fourth $template |"); -my $c = {edges => [], nodes => [ {cmd => [q~/bin/echo~,q~first~,q~second~,q~fourth~,q~fourth~], type => q~EXEC~, id => q~n1~}]}; +my $vtfp_results = from_json(slurp "bin/vtfp.pl -verbosity_level 0 -keys p1,p2,p4 -vals first,second,fourth $template |"); +my $c = {version => q[1.0], edges => [], nodes => [ {cmd => [q~/bin/echo~,q~first~,q~second~,q~fourth~,q~fourth~], type => q~EXEC~, id => q~n1~}]}; is_deeply ($vtfp_results, $c, 'second parameter ring test'); } @@ -47,8 +47,8 @@ is_deeply ($vtfp_results, $c, 'second parameter ring test'); # another variant of the first previous test. It nullifies the first parameter, then sets the value for p5 (in the ring, # but not referenced directly in the node cmd) to confirm that the value propagates through the defaults to p2, p3, and p4 -my $vtfp_results = from_json(slurp "bin/vtfp.pl -nullkeys p1 -keys p5 -vals fifth $template |"); -my $c = {edges => [], nodes => [ {cmd => [q~/bin/echo~,q~fifth~,q~fifth~,q~fifth~], type => q~EXEC~, id => q~n1~}]}; +my $vtfp_results = from_json(slurp "bin/vtfp.pl -verbosity_level 0 -nullkeys p1 -keys p5 -vals fifth $template |"); +my $c = {version => q[1.0], edges => [], nodes => [ {cmd => [q~/bin/echo~,q~fifth~,q~fifth~,q~fifth~], type => q~EXEC~, id => q~n1~}]}; is_deeply ($vtfp_results, $c, 'third parameter ring test'); } diff --git a/t/10-vtfp-pv.t b/t/10-vtfp-pv.t index 34a646211..acd8faa4f 100644 --- a/t/10-vtfp-pv.t +++ b/t/10-vtfp-pv.t @@ -7,8 +7,6 @@ use Perl6::Slurp; use JSON; use File::Temp qw(tempdir); -use Data::Dumper; - my $tdir = tempdir(CLEANUP => 1); my $template = q[t/data/10-vtfp-pv.json]; @@ -26,7 +24,7 @@ subtest 'pv0' => sub { my $vtfp_results = from_json(slurp "bin/vtfp.pl -verbosity_level 0 -param_vals $pv_file $template |"); my $c = from_json(slurp $processed_template); - $expected = {edges=> [], nodes => [ {cmd => [q~/bin/echo~,q~The~,q~funeral~,q~ends~,q~with~,q~a~,q~mournful~,q~fireworks~,q~display~], type => q~EXEC~, id => q~n1~}]}; + $expected = {version => q[1.0], edges=> [], nodes => [ {cmd => [q~/bin/echo~,q~The~,q~funeral~,q~ends~,q~with~,q~a~,q~mournful~,q~fireworks~,q~display~], type => q~EXEC~, id => q~n1~}]}; is_deeply ($vtfp_results, $c, '(ts1) json config generated using pv file matches original generated config'); is_deeply ($vtfp_results, $expected, '(ts1) json config generated using pv file as expected'); }; @@ -42,7 +40,7 @@ subtest 'pv1' => sub { my $vtfp_results = from_json(slurp "bin/vtfp.pl -verbosity_level 0 -param_vals $pv_file $template |"); my $c = from_json(slurp $processed_template); - $expected = {edges=> [], nodes => [ {cmd => [q~/bin/echo~,q~The~,q~party~,q~ends~,q~with~,q~a~,q~deafening~,q~fireworks~,q~display~], type => q~EXEC~, id => q~n1~}]}; + $expected = {version => q[1.0], edges=> [], nodes => [ {cmd => [q~/bin/echo~,q~The~,q~party~,q~ends~,q~with~,q~a~,q~deafening~,q~fireworks~,q~display~], type => q~EXEC~, id => q~n1~}]}; is_deeply ($vtfp_results, $c, '(ts2) json config generated using pv file matches original generated config'); is_deeply ($vtfp_results, $expected, '(ts2) json config generated using pv file as expected'); }; @@ -58,7 +56,7 @@ subtest 'pv2' => sub { my $vtfp_results = from_json(slurp "bin/vtfp.pl -verbosity_level 0 -param_vals $pv_file $template |"); my $c = from_json(slurp $processed_template); - $expected = {edges=> [], nodes => [ {cmd => [q~/bin/echo~,q~The~,q~world~,q~ends~,q~with~,q~a~,q~whimper~], type => q~EXEC~, id => q~n1~}]}; + $expected = {version => q[1.0], edges=> [], nodes => [ {cmd => [q~/bin/echo~,q~The~,q~world~,q~ends~,q~with~,q~a~,q~whimper~], type => q~EXEC~, id => q~n1~}]}; is_deeply ($vtfp_results, $c, '(ts4) json config generated using pv file matches original generated config'); is_deeply ($vtfp_results, $expected, '(ts4) json config generated using pv file as expected'); }; @@ -126,7 +124,9 @@ subtest 'pv3' => sub { my $vtfp_results = from_json(slurp "bin/vtfp.pl -verbosity_level 0 -no-absolute_program_paths -param_vals $pv $template |"); - my $expected = { nodes => + my $expected = { + version => q[1.0], + nodes => [ { id => q[hello], @@ -173,6 +173,7 @@ subtest 'pv3' => sub { $vtfp_results = from_json(slurp "bin/vtfp.pl -verbosity_level 0 -no-absolute_program_paths -param_vals $pv $template |"); $expected = { + version => q[1.0], nodes => [ { diff --git a/t/10-vtfp-select_directive.t b/t/10-vtfp-select_directive.t index 10d07c01c..f7a6cd2b6 100644 --- a/t/10-vtfp-select_directive.t +++ b/t/10-vtfp-select_directive.t @@ -50,6 +50,7 @@ subtest 'select_directive_array' => sub { my $vtfp_results = from_json($test->stdout); my $expected_result = { + version => '1.0', nodes => [ { id => 'n1', @@ -67,6 +68,7 @@ subtest 'select_directive_array' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { id => 'n1', @@ -84,6 +86,7 @@ subtest 'select_directive_array' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { id => 'n1', @@ -139,6 +142,7 @@ subtest 'select_directive_hash' => sub { my $vtfp_results = from_json($test->stdout); my $expected_result = { + version => '1.0', nodes => [ { id => 'n1', @@ -156,6 +160,7 @@ subtest 'select_directive_hash' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { id => 'n1', @@ -173,6 +178,7 @@ subtest 'select_directive_hash' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { id => 'n1', @@ -190,6 +196,7 @@ subtest 'select_directive_hash' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { id => 'n1', @@ -281,6 +288,7 @@ subtest 'select_directive_single_switch_batch_values' => sub { my $vtfp_results = from_json($test->stdout); my $expected_result = { + version => '1.0', nodes => [ { type => "EXEC", @@ -325,6 +333,7 @@ subtest 'select_directive_single_switch_batch_values' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { type => "EXEC", @@ -369,6 +378,7 @@ subtest 'select_directive_single_switch_batch_values' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { type => "EXEC", @@ -480,6 +490,7 @@ subtest 'select_directive_single_switch_batch_values' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { type => "EXEC", @@ -547,6 +558,7 @@ subtest 'select_directive_multi_sel' => sub { my $vtfp_results = from_json($test->stdout); my $expected_result = { + version => '1.0', nodes => [ { type => "EXEC", @@ -585,6 +597,7 @@ subtest 'select_directive_multi_sel' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { type => "EXEC", @@ -625,6 +638,7 @@ subtest 'select_directive_multi_sel' => sub { cmp_ok($exit_status>>8, q(==), 0, "expected exit status of 0 for select with select range - acceptable number of index keys (2, must be between 2 and 3)"); $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { type => "EXEC", @@ -644,6 +658,7 @@ subtest 'select_directive_multi_sel' => sub { cmp_ok($exit_status>>8, q(==), 0, "expected exit status of 0 for select with select range - acceptable number of index keys (3, must be between 2 and 3)"); $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { type => "EXEC", @@ -702,6 +717,7 @@ subtest 'select_directive_multi_sel' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { type => "EXEC", @@ -722,6 +738,7 @@ subtest 'select_directive_multi_sel' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { type => "EXEC", @@ -742,6 +759,7 @@ subtest 'select_directive_multi_sel' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { type => "EXEC", @@ -786,6 +804,7 @@ subtest 'select_directive_no_sel' => sub { my $vtfp_results = from_json($test->stdout); my $expected_result = { + version => '1.0', nodes => [ { type => "EXEC", @@ -806,6 +825,7 @@ subtest 'select_directive_no_sel' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { type => "EXEC", @@ -845,6 +865,7 @@ subtest 'select_directive_no_sel' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { type => "EXEC", @@ -865,6 +886,7 @@ subtest 'select_directive_no_sel' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { type => "EXEC", diff --git a/t/10-vtfp-select_directive.v2.t b/t/10-vtfp-select_directive.v2.t new file mode 100644 index 000000000..cb1cac3e9 --- /dev/null +++ b/t/10-vtfp-select_directive.v2.t @@ -0,0 +1,906 @@ +use strict; +use warnings; +use Carp; +use Test::More tests => 6; +use Test::Cmd; +use File::Slurp; +use Perl6::Slurp; +use JSON; +use File::Temp qw(tempdir); +use Cwd; + +my $tdir = tempdir(CLEANUP => 1); + +my $odir = getcwd(); +my $test = Test::Cmd->new( prog => $odir.'/bin/vtfp.pl', workdir => q()); +ok($test, 'made test object'); + +# simple test of select directive (cases array) +subtest 'select_directive_array' => sub { + plan tests => 8; + + my $select_cmd_template = { + description => 'Test select option, allowing default them explicitly setting the select value', + version => '2.0', + subst_params => [ + { id => 'happy', default => 0 }, + ], + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => { + select => "happy", + cases => + [ + [ 'echo', "bleh" ], + [ 'echo', "woohoo" ], + ], + } + } + ] + }; + + my $template = $tdir.q[/10-vtfp-select_cmd_0_0.json]; + my $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo','bleh',], + } + ], + edges=> [], + }; + + is_deeply ($vtfp_results, $expected_result, 'allow select value to default'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys happy -vals 1 -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo','woohoo',], + } + ], + edges=> [], + }; + + is_deeply ($vtfp_results, $expected_result, 'set select value to non-default'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys happy -vals 0 -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo','bleh',], + } + ], + edges=> [], + }; + + is_deeply ($vtfp_results, $expected_result, 'explicit setting of select value to default'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys happy -vals 16 -verbosity_level 0 $template]); + cmp_ok($exit_status>>8, q(==), 255, "expected exit status of 255 for select with cases array - index out of bounds"); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys happy -vals '-1' -verbosity_level 0 $template]); + cmp_ok($exit_status>>8, q(==), 255, "expected exit status of 255 for select with cases array - negative index"); +}; + +# simple test of select directive (cases hash) +subtest 'select_directive_hash' => sub { + plan tests => 9; + + my $select_cmd_template = { + description => 'Test select option, allowing default them explicitly setting the select value', + version => '2.0', + subst_params => [ + { id => 'mood', default => 'indifferent' }, + ], + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => { + select => "mood", + cases => + { + sad => [ 'echo', "whinge" ], + happy => [ 'echo', "woohoo" ], + indifferent => [ 'echo', "meh" ] + }, + } + } + ] + }; + + my $template = $tdir.q[/10-vtfp-select_cmd_1_0.json]; + my $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo','meh',], + } + ], + edges=> [], + }; + + is_deeply ($vtfp_results, $expected_result, 'allow select value to default'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys mood -vals happy -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo','woohoo',], + } + ], + edges=> [], + }; + + is_deeply ($vtfp_results, $expected_result, 'set select value to non-default'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys mood -vals indifferent -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo','meh',], + } + ], + edges=> [], + }; + + is_deeply ($vtfp_results, $expected_result, 'explicit setting of select value to default'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys mood -vals sad -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo','whinge',], + } + ], + edges=> [], + }; + + is_deeply ($vtfp_results, $expected_result, 'another non-default setting of select value'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys mood -vals ecstatic -verbosity_level 0 $template]); + cmp_ok($exit_status>>8, q(==), 255, "expected exit status of 255 for select with cases hash - index key not present in cases"); +}; + +# single switch change batch of values +subtest 'select_directive_single_switch_batch_values' => sub { + plan tests => 8; + + my $select_cmd_template = { + version => "2.0", + description => "batch of values assigned using one select value", + subst_params => [ + { + select => "mslang", default => "eng", + cases => { + eng => [ + { id => "valA", default => "one" }, + { id => "valB", default => "two" }, + { id => "valC", default => "three" } + ], + deu => [ + { id => "valA", default => "eins" }, + { id => "valB", default => "zwei" }, + { id => "valC", default => "drei" } + ], + spa => [ + { id => "valA", default => "uno" }, + { id => "valB", default => "dos" }, + { id => "valC", default => "tres" } + ] + } + } + ], + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {subst => "valA", ifnull => "one"} ] + }, + { + id => "B", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {subst => "valB", ifnull => "two"} ] + }, + { + id => "C", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {subst => "valC", ifnull => "three"} ] + }, + { + id => "P", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "paste", {"port" => "A", "direction" => "in"}, {"port" => "B", "direction" => "in"}, {"port" => "C", "direction" => "in"} ] + } + ], + edges => [ + { id => "AP", from => "A", to => "P:A" }, + { id => "BP", from => "B", to => "P:B" }, + { id => "CP", from => "C", to => "P:C" } + ] + }; + + my $template = $tdir.q[/10-vtfp-select_cmd_2_0.json]; + my $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "one" ], + use_STDIN => JSON::false, + id => "A" + }, + { + use_STDIN => JSON::false, + cmd => [ "echo", "two" ], + id => "B", + type => "EXEC", + use_STDOUT => JSON::true + }, + { + id => "C", + cmd => [ "echo", "three" ], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + type => "EXEC" + }, + { + id => "P", + use_STDIN => JSON::false, + cmd => [ "paste", {"port" => "A", "direction" => "in"}, {"port" => "B", "direction" => "in"}, {"port" => "C","direction" => "in"} ], + use_STDOUT => JSON::true, + type => "EXEC" + } + ], + edges => [ + { from => "A", to => "P:A", id => "AP" }, + { from => "B", id => "BP", to => "P:B" }, + { from => "C", id => "CP", to => "P:C" } + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'single switch/batch of values with default'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys mslang -vals deu -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "eins" ], + use_STDIN => JSON::false, + id => "A" + }, + { + use_STDIN => JSON::false, + cmd => [ "echo", "zwei" ], + id => "B", + type => "EXEC", + use_STDOUT => JSON::true + }, + { + id => "C", + cmd => [ "echo", "drei" ], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + type => "EXEC" + }, + { + id => "P", + use_STDIN => JSON::false, + cmd => [ "paste", {"port" => "A", "direction" => "in"}, {"port" => "B","direction" => "in"}, {"port" => "C","direction" => "in"} ], + use_STDOUT => JSON::true, + type => "EXEC" + } + ], + edges => [ + { from => "A", to => "P:A", id => "AP" }, + { from => "B", id => "BP", to => "P:B" }, + { from => "C", id => "CP", to => "P:C" } + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'single switch changes batch of values, select deu'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys mslang -vals spa -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "uno" ], + use_STDIN => JSON::false, + id => "A" + }, + { + use_STDIN => JSON::false, + cmd => [ "echo", "dos" ], + id => "B", + type => "EXEC", + use_STDOUT => JSON::true + }, + { + id => "C", + cmd => [ "echo", "tres" ], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + type => "EXEC" + }, + { + id => "P", + use_STDIN => JSON::false, + cmd => [ "paste", {"port" => "A", "direction" => "in"}, {"port" => "B","direction" => "in"}, {"port" => "C","direction" => "in"} ], + use_STDOUT => JSON::true, + type => "EXEC" + } + ], + edges => [ + { from => "A", to => "P:A", id => "AP" }, + { from => "B", id => "BP", to => "P:B" }, + { from => "C", id => "CP", to => "P:C" } + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'single switch/batch of values, select spa'); + + # same basic template as above, but without a default value for the select + $select_cmd_template = { + version => "2.0", + description => "batch of values assigned using one select value (no default)", + subst_params => [ + { + select => "mslang", + cases => { + eng => [ + { id => "valA", default => "one" }, + { id => "valB", default => "two" }, + { id => "valC", default => "three" } + ], + deu => [ + { id => "valA", default => "eins" }, + { id => "valB", default => "zwei" }, + { id => "valC", default => "drei" } + ], + spa => [ + { id => "valA", default => "uno" }, + { id => "valB", default => "dos" }, + { id => "valC", default => "tres" } + ] + } + } + ], + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {subst => "valA", ifnull => "unspec"} ] + }, + { + id => "B", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {subst => "valB", ifnull => "unspec"} ] + }, + { + id => "C", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {subst => "valC", ifnull => "unspec"} ] + }, + { + id => "P", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => ["paste", {"port" => "A","direction" => "in"}, {"port" => "B", "direction" => "in"}, {"port" => "C","direction" => "in"} ] + } + ], + edges => [ + { id => "AP", from => "A", to => "P:A" }, + { id => "BP", from => "B", to => "P:B" }, + { id => "CP", from => "C", to => "P:C" } + ] + }; + + $template = $tdir.q[/10-vtfp-select_cmd_2_1.json]; + $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "unspec" ], + use_STDIN => JSON::false, + id => "A" + }, + { + use_STDIN => JSON::false, + cmd => [ "echo", "unspec" ], + id => "B", + type => "EXEC", + use_STDOUT => JSON::true + }, + { + id => "C", + cmd => [ "echo", "unspec" ], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + type => "EXEC" + }, + { + id => "P", + use_STDIN => JSON::false, + cmd => [ "paste", {"port" => "A","direction" => "in"}, {"port" => "B","direction" => "in"}, {"port" => "C", "direction" => "in"} ], + use_STDOUT => JSON::true, + type => "EXEC" + } + ], + edges => [ + { from => "A", to => "P:A", id => "AP" }, + { from => "B", id => "BP", to => "P:B" }, + { from => "C", id => "CP", to => "P:C" } + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'single switch/batch of values without default'); +}; + +# multisel +subtest 'select_directive_multi_sel' => sub { + plan tests => 16; + + my $select_cmd_template = { + version => "2.0", + description => "select multiple values from cases (array)", + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {select => "wordsel", cases => [ "one", "two", "three", "four" ]} ] + }, + ], + }; + + my $template = $tdir.q[/10-vtfp-select_cmd_3_0.json]; + my $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys wordsel,wordsel -vals 1,3 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "two", "four" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select multiple values from cases (array)'); + + $select_cmd_template = { + version => "2.0", + description => "select multiple values from cases (hash)", + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {select => "wordsel", cases => { first => "one", second => "two", third => "three", fourth => "four" }} ] + }, + ], + }; + + $template = $tdir.q[/10-vtfp-select_cmd_3_1.json]; + $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys wordsel,wordsel -vals second,fourth $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "two", "four" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select multiple values from cases (hash)'); + + $select_cmd_template = { + version => "2.0", + description => "select multiple values from cases (array)", + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {select => "wordsel", "select_range" => [2,3], cases => [ "one", "two", "three", "four" ]} ] + }, + ], + }; + + $template = $tdir.q[/10-vtfp-select_cmd_3_2.json]; + $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys wordsel -vals 1 $template]); + cmp_ok($exit_status>>8, q(==), 255, "expected exit status of 255 for select with select range - too few index keys (only 1, must be between 2 and 3)"); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys wordsel,wordsel -vals 1,2 $template]); + cmp_ok($exit_status>>8, q(==), 0, "expected exit status of 0 for select with select range - acceptable number of index keys (2, must be between 2 and 3)"); + $vtfp_results = from_json($test->stdout); + $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "two", "three" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'valid select with select_range (2 with 2-3)'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys wordsel,wordsel,wordsel -vals 0,1,2 $template]); + cmp_ok($exit_status>>8, q(==), 0, "expected exit status of 0 for select with select range - acceptable number of index keys (3, must be between 2 and 3)"); + $vtfp_results = from_json($test->stdout); + $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "one", "two", "three" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'valid select with select_range (3 with 2-3)'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys wordsel,wordsel,wordsel,wordsel -vals 0,1,2,3 $template]); + cmp_ok($exit_status>>8, q(==), 255, "expected exit status of 255 for select with select range - too few index keys (4, must be between 2 and 3)"); + + $select_cmd_template = { + version => "2.0", + description => "select multiple non-unique values from cases (array)", + subst_params => [ + { + id => "notes", + default => { + select => "tune", + cases => { + happybirthday => [ 4, 4, 5, 4, 0, 6, 4, 4, 5, 4, 1, 0, 4, 4, 4, 2, 1, 0, 6, 2, 2, 1, 0, 1, 0 ], + scale => [ 0, 1, 2, 3, 4, 5, 6 ] + } + } + } + ], + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", + { select => "notes", + cases => [ "do", "re", "mi", "fa", "so", "la", "ti" ], + default => [ 0, 2, 2, 2, 4, 4, 1, 3, 3, 5, 6, 6 ] + } + ], + }, + ], + }; + + $template = $tdir.q[/10-vtfp-select_cmd_3_3.json]; + $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "do", "mi", "mi", "mi", "so", "so", "re", "fa", "fa", "la", "ti", "ti" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select multiple non-unique values from cases (array)'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys tune -vals happybirthday $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "so", "so", "la", "so", "do", "ti", "so", "so", "la", "so", "re", "do", "so", "so", "so", "mi", "re", "do", "ti", "mi", "mi", "re", "do", "re", "do" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select multiple non-unique values from cases (array) (hb)'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys tune -vals scale $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "do", "re", "mi", "fa", "so", "la", "ti" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select multiple non-unique values from cases (array) (scale)'); +}; + +### no_sel - explicitly switch off +subtest 'select_directive_no_sel' => sub { + plan tests => 8; + + # with array cases + my $select_cmd_template = { + version => "2.0", + description => "select multiple values from cases (array)", + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {select => "wordsel", "default" => 3, cases => [ "one", "two", "three", "four" ]} ] + }, + ], + }; + + my $template = $tdir.q[/10-vtfp-select_cmd_4_0.json]; + my $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "four" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select value using default from cases (array)'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys fred -vals bloggs -nullkeys wordsel $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select no values from cases (array), overriding default with nullkeys'); + + # now the same but with hash cases + $select_cmd_template = { + version => "2.0", + description => "select multiple values from cases (array)", + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {select => "wordsel", "default" => "fourth", cases => { first => "one", second => "two", third => "three", fourth => "four" }} ] + }, + ], + }; + + $template = $tdir.q[/10-vtfp-select_cmd_4_1.json]; + $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "four" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select value using default from cases (hash)'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys fred -vals bloggs -nullkeys wordsel $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select no values from cases (hash), overriding default with nullkeys'); +}; + +1; diff --git a/t/10-vtfp-splice_nodes.t b/t/10-vtfp-splice_nodes.t index 1d6331848..16a87d370 100644 --- a/t/10-vtfp-splice_nodes.t +++ b/t/10-vtfp-splice_nodes.t @@ -1,7 +1,7 @@ use strict; use warnings; use Carp; -use Test::More tests => 4; +use Test::More tests => 6; use Test::Cmd; use File::Slurp; use Perl6::Slurp; @@ -248,7 +248,9 @@ subtest 'spl0' => sub { write_file($template, $template_contents); my $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -splice_nodes rev $template |"); - my $expected = { nodes => + my $expected = { + version => q[1.0], + nodes => [ { id => q[hello], @@ -288,7 +290,9 @@ subtest 'spl0' => sub { is_deeply ($vtfp_results, $expected, '(spl0) one node in a chain spliced out'); $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -splice_nodes \'uc-\' $template |"); - $expected = { nodes => + $expected = { + version => q[1.0], + nodes => [ { id => q[hello], @@ -324,7 +328,9 @@ subtest 'spl0' => sub { write_file($template, $template_contents); $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -splice_nodes rev $template |"); - $expected = { nodes => + $expected = { + version => q[1.0], + nodes => [ { id => q[hello], @@ -364,7 +370,9 @@ subtest 'spl0' => sub { is_deeply ($vtfp_results, $expected, '(spl0) one node in a chain (including edges with no ids) spliced out'); $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -splice_nodes \'uc-\' $template |"); - $expected = { nodes => + $expected = { + version => q[1.0], + nodes => [ { id => q[hello], @@ -406,6 +414,7 @@ subtest 'pru0' => sub { my $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -prune_nodes disemvowel- $template |"); my $expected = { + version => q[1.0], nodes => [ { @@ -445,6 +454,7 @@ subtest 'pru0' => sub { $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -prune_nodes disemvowel- $template |"); $expected = { + version => q[1.0], nodes => [ { @@ -484,6 +494,7 @@ subtest 'pru0' => sub { $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -prune_nodes \'-P:__A_IN__;-P:__C_IN__\' $template |"); $expected = { + version => q[1.0], nodes => [ { @@ -516,6 +527,7 @@ subtest 'pru0' => sub { $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -prune_nodes \'-P:__(A|C)_IN__\' $template |"); $expected = { + version => q[1.0], nodes => [ { @@ -548,6 +560,7 @@ subtest 'pru0' => sub { $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -prune_nodes output $template |"); $expected = { + version => q[1.0], nodes => [ { @@ -601,6 +614,7 @@ subtest 'spl1' => sub { my $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -splice_nodes \'.*[rs]e.*\' $template |"); my $expected = { + version => q[1.0], nodes => [ { @@ -634,6 +648,7 @@ subtest 'spl1' => sub { $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -prune_nodes \'disemv.*-\' $template |"); $expected = { + version => q[1.0], nodes => [ { @@ -673,6 +688,7 @@ subtest 'spl1' => sub { $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -prune_nodes \'tee2-;tee3-\' $template |"); $expected = { + version => q[1.0], nodes => [ { @@ -708,6 +724,7 @@ subtest 'spl1' => sub { $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -prune_nodes \'tee.*:__RIGHT.*-\' $template |"); $expected = { + version => q[1.0], nodes => [ { @@ -799,4 +816,598 @@ subtest 'spl2' => sub { }; +my $basic_linear_template_v2 = { + description => q[simple linear chain (stdin->stdout) of nodes], + version => q[2.0], + nodes => + [ + { + id => q[hello], + type => q[EXEC], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ q/echo/, q/Hello/ ], + }, + { + id => q[rev], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/rev/ ] + }, + { + id => q[uc], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/tr/, q/[:lower:]/, q/[:upper:]/ ], + }, + { + id => q[disemvowel], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/tr/, q/-d/, q/[aeiouAEIOU]/ ], + }, + { + id => q[output], + type => q[OUTFILE], + name => q/tmp.xxx/, + }, + ], + edges => + [ + { id => q[e0], from => q[hello], to => q[rev] }, + { id => q[e1], from => q[rev], to => q[uc] }, + { id => q[e2], from => q[uc], to => q[disemvowel] }, + { id => q[e3], from => q[disemvowel], to => q[output] } + ] +}; + +my $basic_linear_template_missing_edge_ids_v2 = { + description => q[simple linear chain (stdin->stdout) of nodes], + version => q[2.0], + nodes => + [ + { + id => q[hello], + type => q[EXEC], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ q/echo/, q/Hello/ ], + }, + { + id => q[rev], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/rev/ ] + }, + { + id => q[uc], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/tr/, q/[:lower:]/, q/[:upper:]/ ], + }, + { + id => q[disemvowel], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/tr/, q/-d/, q/[aeiouAEIOU]/ ], + }, + { + id => q[output], + type => q[OUTFILE], + name => q/tmp.xxx/, + }, + ], + edges => + [ + { id => q[e0], from => q[hello], to => q[rev] }, + { from => q[rev], to => q[uc] }, + { id => q[e2], from => q[uc], to => q[disemvowel] }, + { from => q[disemvowel], to => q[output] } + ] +}; + +my $basic_multipath_template_v2 = { + description => q[graph with some branching], + version => q[2.0], + nodes => + [ + { + id => q[top], + type => q[EXEC], + subtype => q[STRINGIFY], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", "Start", "|", "tee", {port => "ALT", direction => "out"} ] + }, + { + id => q[nodeA], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/sed/, q/-e/, q#s/$/ addedA\\n/#, ] + }, + { + id => q[nodeB], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/sed/, q/-e/, q#s/$/ addedB\\n/#, ] + }, + { + id => q[tee2], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/tee/, {port => "RIGHT2", direction => "out"} ] + }, + { + id => q[tee3], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/tee/, {port => "RIGHT3", direction => "out"}, {port => "MID3", direction => "out"} ] + }, + { + id => q[Aleft], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/sed/, q/-e/, q#s/$/ left\\n/# ], + }, + { + id => q[Aright], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/sed/, q/-e/, q#s/$/ right\\n/# ], + }, + { + id => q[Bleft], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/sed/, q/-e/, q#s/$/ left\\n/# ], + }, + { + id => q[Bmid], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/sed/, q/-e/, q#s/$/ mid\\n/# ], + }, + { + id => q[Bright], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/sed/, q/-e/, q#s/$/ right\\n/# ], + }, + ], + edges => + [ + { id => q[e0], from => q[top], to => q[nodeA] }, + { id => q[e1], from => q[top:ALT], to => q[nodeB] }, + { id => q[e2], from => q[nodeA], to => q[tee2] }, + { id => q[e3], from => q[nodeB], to => q[tee3] }, + { id => q[e4], from => q[tee2], to => q[Aleft] }, + { id => q[e5], from => q[tee2:RIGHT2], to => q[Aright] }, + { id => q[e6], from => q[tee3], to => q[Bleft] }, + { id => q[e7], from => q[tee3:MID3], to => q[Bmid] }, + { id => q[e8], from => q[tee3:RIGHT3], to => q[Bright] }, + ] +}; + +my $multi_src_template_v2 = { + version => q[2.0], + description => q[graph with multiple sources], + nodes => + [ + { + id => q[A], + type => q[EXEC], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ 'echo', 'A', ], + }, + { + id => q[B], + type => q[EXEC], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ 'echo', 'B', ], + }, + { + id => q[C], + type => q[EXEC], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ 'echo', 'C', ], + }, + { + id => q[P], + type => q[EXEC], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ 'paste', {port => 'A', direction => 'in'}, {port => 'B', direction => 'in'}, {port => 'C', direction => 'in'}, ], + } + ], + edges => + [ + { id => q/AP/, from => q/A/, to => q/P:A/, }, + { id => q/BP/, from => q/B/, to => q/P:B/, }, + { id => q/CP/, from => q/C/, to => q/P:C/, }, + ], +}; + +subtest 'spl00' => sub { + plan tests => 4; + + my $template = $tdir.q[/10-vtfp-splice_nodes_00.json]; + my $template_contents = to_json($basic_linear_template_v2); + write_file($template, $template_contents); + + my $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -splice_nodes rev $template |"); + my $expected = { + version => q[2.0], + nodes => + [ + { + id => q[hello], + type => q[EXEC], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ q/echo/, q/Hello/ ], + }, + { + id => q[uc], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/tr/, q/[:lower:]/, q/[:upper:]/ ], + }, + { + id => q[disemvowel], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/tr/, q/-d/, q/[aeiouAEIOU]/ ], + }, + { + id => q[output], + type => q[OUTFILE], + name => q/tmp.xxx/, + }, + ], + edges => + [ + { id => q/e2/, from => q/uc/, to => q/disemvowel/ }, + { id => q/e3/, from => q/disemvowel/, to => q/output/ }, + { id => q/hello_to_uc/, from => q/hello/, to => q/uc/ }, + ] + }; + + is_deeply ($vtfp_results, $expected, '(spl00) one node in a chain spliced out'); + + $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -splice_nodes \'uc-\' $template |"); + $expected = { + version => q[2.0], + nodes => + [ + { + id => q[hello], + type => q[EXEC], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ q/echo/, q/Hello/ ], + }, + { + id => q[rev], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/rev/ ], + }, + { + id => q[STDOUT_00000], + name => q[/dev/stdout], + type => q[OUTFILE], + }, + ], + edges => + [ + { id => q/e0/, from => q/hello/, to => q/rev/ }, + { id => q/rev_to_STDOUT/, from => q/rev/, to => q/STDOUT_00000/ }, + ] + }; + + is_deeply ($vtfp_results, $expected, '(spl00) remove all nodes from uc node downstream in the chain (output to STDIN'); + + $template = $tdir.q[/10-vtfp-splice_nodes_01.json]; + $template_contents = to_json($basic_linear_template_missing_edge_ids_v2); + write_file($template, $template_contents); + + $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -splice_nodes rev $template |"); + $expected = { + version => q[2.0], + nodes => + [ + { + id => q[hello], + type => q[EXEC], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ q/echo/, q/Hello/ ], + }, + { + id => q[uc], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/tr/, q/[:lower:]/, q/[:upper:]/ ], + }, + { + id => q[disemvowel], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/tr/, q/-d/, q/[aeiouAEIOU]/ ], + }, + { + id => q[output], + type => q[OUTFILE], + name => q/tmp.xxx/, + }, + ], + edges => + [ + { id => q/e2/, from => q/uc/, to => q/disemvowel/ }, + { from => q/disemvowel/, to => q/output/ }, + { id => q/hello_to_uc/, from => q/hello/, to => q/uc/ }, + ] + }; + + is_deeply ($vtfp_results, $expected, '(spl00) one node in a chain (including edges with no ids) spliced out'); + + $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -splice_nodes \'uc-\' $template |"); + $expected = { + version => q[2.0], + nodes => + [ + { + id => q[hello], + type => q[EXEC], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ q/echo/, q/Hello/ ], + }, + { + id => q[rev], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/rev/ ], + }, + { + id => q[STDOUT_00000], + name => q[/dev/stdout], + type => q[OUTFILE], + }, + ], + edges => + [ + { id => q/e0/, from => q/hello/, to => q/rev/ }, + { id => q/rev_to_STDOUT/, from => q/rev/, to => q/STDOUT_00000/ }, + ] + }; + + is_deeply ($vtfp_results, $expected, '(spl00) remove all nodes from uc node downstream in a chain including edges with no ids (output to STDIN'); +}; + + +# prune tests +subtest 'pru00' => sub { + plan tests => 5; + + my $template = $tdir.q[/10-vtfp-prune_nodes_00.json]; + my $template_contents = to_json($basic_linear_template_v2); + write_file($template, $template_contents); + + my $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -prune_nodes disemvowel- $template |"); + my $expected = { + version => q[2.0], + nodes => + [ + { + id => q[hello], + type => q[EXEC], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ q/echo/, q/Hello/ ], + }, + { + id => q[rev], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/rev/ ], + }, + { + id => q[uc], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::false, + cmd => [ q/tr/, q/[:lower:]/, q/[:upper:]/ ], + }, + ], + edges => + [ + { id => q/e0/, from => q/hello/, to => q/rev/ }, + { id => q/e1/, from => q/rev/, to => q/uc/ }, + ] + }; + + is_deeply ($vtfp_results, $expected, '(pru00.0) prune final two nodes in a chain (output to STDOUT switched off)'); + + $template = $tdir.q[/10-vtfp-prune_nodes_01.json]; + $template_contents = to_json($basic_linear_template_missing_edge_ids_v2); + write_file($template, $template_contents); + + $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -prune_nodes disemvowel- $template |"); + $expected = { + version => q[2.0], + nodes => + [ + { + id => q[hello], + type => q[EXEC], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ q/echo/, q/Hello/ ], + }, + { + id => q[rev], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/rev/ ], + }, + { + id => q[uc], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::false, + cmd => [ q/tr/, q/[:lower:]/, q/[:upper:]/ ], + }, + ], + edges => + [ + { id => q/e0/, from => q/hello/, to => q/rev/ }, + { from => q/rev/, to => q/uc/ }, + ] + }; + + is_deeply ($vtfp_results, $expected, '(pru00.0) prune final two nodes in a chain including edge elements with no id (output to STDOUT switched off)'); + + $template = $tdir.q[/10-vtfp-prune_nodes_02.json]; + $template_contents = to_json($multi_src_template_v2); + write_file($template, $template_contents); + + $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -prune_nodes \'-P:A;-P:C\' $template |"); + $expected = { + version => q[2.0], + nodes => + [ + { + id => q[B], + type => q[EXEC], + use_STDOUT => JSON::true, + use_STDIN => JSON::false, + cmd => [ 'echo', 'B', ], + }, + { + id => q[P], + type => q[EXEC], + use_STDOUT => JSON::true, + use_STDIN => JSON::false, + cmd => [ 'paste', {port => 'B', direction => 'in'}, ], + }, + ], + edges => + [ + { id => 'BP', from => 'B', to => 'P:B' } + + ] + }; + + is_deeply ($vtfp_results, $expected, '(pru00.1) remove two branches specified using implied STDIN src in prune spec'); + + $template = $tdir.q[/10-vtfp-prune_nodes_03.json]; + $template_contents = to_json($multi_src_template_v2); + write_file($template, $template_contents); + + $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -prune_nodes \'-P:(A|C)\' $template |"); + $expected = { + version => q[2.0], + nodes => + [ + { + id => q[B], + type => q[EXEC], + use_STDOUT => JSON::true, + use_STDIN => JSON::false, + cmd => [ 'echo', 'B', ], + }, + { + id => q[P], + type => q[EXEC], + use_STDOUT => JSON::true, + use_STDIN => JSON::false, + cmd => [ 'paste', {port => 'B', direction => 'in'}, ], + }, + ], + edges => + [ + { id => 'BP', from => 'B', to => 'P:B' } + + ] + }; + + is_deeply ($vtfp_results, $expected, '(pru00.2) remove two branches specified using implied STDIN src in prune spec (pru0.1 with regexp prune spec)'); + + $template = $tdir.q[/10-vtfp-prune_nodes_04.json]; + $template_contents = to_json($basic_linear_template_v2); + write_file($template, $template_contents); + + $vtfp_results = from_json(slurp "bin/vtfp.pl -no-absolute_program_paths -verbosity_level 0 -prune_nodes output $template |"); + $expected = { + version => q[2.0], + nodes => + [ + { + id => q[hello], + type => q[EXEC], + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ q/echo/, q/Hello/ ], + }, + { + id => q[rev], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/rev/ ], + }, + { + id => q[uc], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::true, + cmd => [ q/tr/, q/[:lower:]/, q/[:upper:]/ ], + }, + { + id => q[disemvowel], + type => q[EXEC], + use_STDIN => JSON::true, + use_STDOUT => JSON::false, + cmd => [ q/tr/, q/-d/, q/[aeiouAEIOU]/ ], + }, + ], + edges => + [ + { id => q/e0/, from => q/hello/, to => q/rev/ }, + { id => q/e1/, from => q/rev/, to => q/uc/ }, + { id => q[e2], from => q[uc], to => q[disemvowel] }, + ] + }; + + is_deeply ($vtfp_results, $expected, '(pru00.3) prune final file output node (no range specified; output to STDOUT switched off)'); + +}; + 1; diff --git a/t/10-vtfp-subst_directive.t b/t/10-vtfp-subst_directive.t index a00f4ed43..679a87d4c 100644 --- a/t/10-vtfp-subst_directive.t +++ b/t/10-vtfp-subst_directive.t @@ -48,6 +48,7 @@ subtest 'flat_array_param_names' => sub { my $vtfp_results = from_json($test->stdout); my $expected_result = { + version => '1.0', nodes => [ { id => 'n1', @@ -102,6 +103,7 @@ subtest 'indirect_param_names' => sub { my $vtfp_results = from_json($test->stdout); my $expected_result = { + version => '1.0', nodes => [ { id => 'fred', @@ -127,6 +129,7 @@ subtest 'indirect_param_names' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { id => 'fred', @@ -192,6 +195,7 @@ subtest 'full_node_subst' => sub { my $vtfp_results = from_json($test->stdout); my $expected_result = { + version => '1.0', nodes => [ { id => 'apn0', @@ -257,6 +261,7 @@ subtest 'subst_containing_subst' => sub { my $vtfp_results = from_json($test->stdout); my $expected_result = { + version => '1.0', nodes => [ { id => 'node0', @@ -306,6 +311,7 @@ subtest 'required_subst' => sub { my $vtfp_results = from_json($test->stdout); my $expected_result = { + version => '1.0', nodes => [ { id => 'node0', @@ -361,6 +367,7 @@ subtest 'subst_locality' => sub { my $vtfp_results = from_json($test->stdout); my $expected_result = { + version => '1.0', nodes => [ { id => 'node0', @@ -393,6 +400,7 @@ subtest 'subst_locality' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { id => 'node0', @@ -471,6 +479,7 @@ subtest 'cmd_pp' => sub { my $vtfp_results = from_json($test->stdout); my $expected_result = { + version => '1.0', nodes => [ { id => 'node0', @@ -495,6 +504,7 @@ subtest 'cmd_pp' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { id => 'node0', diff --git a/t/10-vtfp-subst_directive.v2.t b/t/10-vtfp-subst_directive.v2.t new file mode 100644 index 000000000..5c05634d5 --- /dev/null +++ b/t/10-vtfp-subst_directive.v2.t @@ -0,0 +1,535 @@ +use strict; +use warnings; +use Carp; +use Test::More tests => 8; +use Test::Cmd; +use File::Slurp; +use Perl6::Slurp; +use JSON; +use File::Temp qw(tempdir); +use Cwd; + +my $tdir = tempdir(CLEANUP => 1); + +my $odir = getcwd(); +my $test = Test::Cmd->new( prog => $odir.'/bin/vtfp.pl', workdir => q()); +ok($test, 'made test object'); + +# Test expansion of parameters, specifically cases where there are multiple instances of multi-valued parameters (arrays). +# In the cmd attribute, the result should be an array of scalars +subtest 'flat_array_param_names' => sub { + plan tests => 2; + + my $flat_array_param_names_template = { + description => 'Test expansion of parameters, specifically cases where there are multiple instances of multi-valued parameters (arrays)', + version => '2.0', + subst_params => [ + { id => 'p1', subst_constructor => { vals => [ '1A', '1B' ] } }, + { id => 'p2', subst_constructor => { vals => [ '2A' ] } }, + { id => 'p3', subst_constructor => { vals => [ '3A', '3B', '3C', '3D' ] } }, + { id => 'p4', default => '4A'}, + { id => 'p5', subst_constructor => { vals => [ '5A', '5B' ] } } + ], + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => [ 'echo', {subst => 'p1'}, {subst => 'p2'}, {subst => 'p3'}, {subst => 'p4'}, {subst => 'p5'} ] + } + ] + }; + + my $template = $tdir.q[/10-vtfp-select_0_0.json]; + my $template_contents = to_json($flat_array_param_names_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo','1A','1B','2A','3A','3B','3C','3D','4A','5A','5B'], + } + ], + edges=> [], + }; + + is_deeply ($vtfp_results, $expected_result, 'flat array param_names results comparison'); +}; + +# subst directive name requires evaluation (is not just a string) +subtest 'indirect_param_names' => sub { + plan tests => 4; + + my $indirect_param_names_template = { + description => q/indirect parameter names examples/, + version => q/2.0/, + nodes => + [ + { + id => q/fred/, + description => q/parameter name is the value of another parameter/, + type => q/EXEC/, + cmd => [ + q/echo/, + q/fredvalue:/, + { subst => { subst => q/alice/ } } + ], + }, + { + id => q/bill/, + description => q/parameter name is the result of a subst_constructor evaluation/, + type => q/EXEC/, + cmd => [ + q/echo/, + q/billval:/, + { subst => { subst_constructor => { vals => [ q/b/, q/i/, q/l/, q/l/ ], postproc => { op => q/concat/, pad => q//, } } } } + ], + }, + ], + }; + + my $template = $tdir.q[/10-vtfp-select_1_0.json]; + my $template_contents = to_json($indirect_param_names_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys fred,bill,alice -vals bloggs,bailey,fred $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + id => 'fred', + description => 'parameter name is the value of another parameter', + type => 'EXEC', + cmd => [ 'echo', 'fredvalue:', 'bloggs' ], + }, + { + id => 'bill', + description => 'parameter name is the result of a subst_constructor evaluation', + type => 'EXEC', + cmd => [ 'echo', 'billval:', 'bailey' ], + } + ], + edges => [], + }; + + is_deeply ($vtfp_results, $expected_result, 'vtfp results 1 comparison'); + + # change parameter values + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys fred,bill,alice -vals bloggs,bailey,bill $template]); + ok($exit_status>>8 == 0, "non-zero exit for test2: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + id => 'fred', + description => 'parameter name is the value of another parameter', + type => 'EXEC', + cmd => [ 'echo', 'fredvalue:', 'bailey' ], + }, + { + id => 'bill', + description => 'parameter name is the result of a subst_constructor evaluation', + type => 'EXEC', + cmd => [ 'echo', 'billval:', 'bailey' ], + } + ], + edges => [], + }; + + is_deeply ($vtfp_results, $expected_result, 'vtfp results 2 comparison'); + +}; + +# entire node is subst directive +subtest 'full_node_subst' => sub { + plan tests => 2; + + my $full_nodes_as_subst_template = { + version => "2.0", + description => "subst directive name requires evaluation (is not just a string)", + subst_params => [ + { id => "fred", default => "Bloggs" }, + { id => "alice", default => "fred" }, + { + id => "all_param_node0", + default => { + id => "apn0", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", "subst", "from", "sp", "section" ], + }, + }, + ], + nodes => [ + { subst => "all_param_node0" }, + { subst => "all_param_node1", + ifnull => { + id => "apn1", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", "subst", "from", "ifnull" ], + }, + }, + ], + }; + + my $template = $tdir.q[/10-vtfp-select_2_0.json]; + my $template_contents = to_json($full_nodes_as_subst_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + id => 'apn0', + type => 'EXEC', + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ + 'echo', + 'subst', + 'from', + 'sp', + 'section' + ], + }, + { + id => 'apn1', + type => 'EXEC', + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ + 'echo', + 'subst', + 'from', + 'ifnull' + ], + } + ], + edges => [], + }; + + is_deeply ($vtfp_results, $expected_result, 'entire node is subst directive results comparison'); +}; + +# confirm that nested subst directives work +subtest 'subst_containing_subst' => sub { + plan tests => 2; + + my $nested_subst_template = { + version => '2.0', + description => 'subst nesting', + subst_params => [ + { id => 'node0_cmd', default => [ 'echo', 'Hello from node0', { subst => 'node0_extra_ms', }, ], + }, + { id => "node0_extra_ms", default => ' plus something extra' }, + ], + nodes => [ + { + id => 'node0', + type => 'EXEC', + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => { subst => 'node0_cmd' } + }, + ], + }; + + my $template = $tdir.q[/10-vtfp-nested_subst_template.json]; + my $template_contents = to_json($nested_subst_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + id => 'node0', + type => 'EXEC', + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ + 'echo', + 'Hello from node0', + ' plus something extra', + ], + }, + ], + edges => [], + }; + + is_deeply ($vtfp_results, $expected_result, 'entire node is subst directive results comparison'); +}; + +# check behaviour of required attribute +subtest 'required_subst' => sub { + plan tests => 3; + + my $required_subst_template = { + version => '2.0', + description => 'subst: required attrib testing', + nodes => [ + { + id => 'node0', + type => 'EXEC', + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {subst => 'alice', required => JSON::true, }, {subst => 'bob', ifnull => 'Robert' }, {subst => 'carol', required => 'no', },] + }, + ], + }; + + my $template = $tdir.q[/10-vtfp-required_subst_template.json]; + my $template_contents = to_json($required_subst_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + cmp_ok($exit_status>>8, q(==), 255, "expected exit status of 255 for missing required test"); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys alice -vals Albert $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + id => 'node0', + type => 'EXEC', + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ + 'echo', + 'Albert', + 'Robert', + ], + }, + ], + edges => [], + }; + + is_deeply ($vtfp_results, $expected_result, 'required subst directive'); +}; + +# confirm local effect of subst directive +subtest 'subst_locality' => sub { + plan tests => 4; + + my $subst_locality_template = { + version => '2.0', + description => 'subst: locality testing', + subst_params => [ + { id => 'cartoon', default => 'Flintstone'} , + ], + nodes => [ + { + id => 'node0', + type => 'EXEC', + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ 'echo', + 'Four substs of one fred parameter', + '1:', {subst => 'fred', ifnull => 'Bloggs', }, + '2:', {subst => 'fred', }, + '3:', {subst => 'fred', required => JSON::true, ifnull => {subst => 'cartoon'}, }, + '4:', {subst => 'fred', required => JSON::true, ifnull => {subst => 'cartoon'}, }, + '5:', {subst => 'fred', ifnull => 'V', }, ] + }, + ], + }; + + my $template = $tdir.q[/10-vtfp-subst_locality_template.json]; + my $template_contents = to_json($subst_locality_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + id => 'node0', + type => 'EXEC', + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ + 'echo', + 'Four substs of one fred parameter', + '1:', + 'Bloggs', + '2:', + '3:', + 'Flintstone', + '4:', + 'Flintstone', + '5:', + 'V', + ], + }, + ], + edges => [], + }; + + is_deeply ($vtfp_results, $expected_result, 'subst locality'); + + # now set a parameter from the command line + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys cartoon -vals Jetson $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + id => 'node0', + type => 'EXEC', + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ + 'echo', + 'Four substs of one fred parameter', + '1:', + 'Bloggs', + '2:', + '3:', + 'Jetson', + '4:', + 'Jetson', + '5:', + 'V', + ], + }, + ], + edges => [], + }; + + is_deeply ($vtfp_results, $expected_result, 'subst locality'); +}; + +# cmd strings postprocessing produces a string or an array of strings +subtest 'cmd_pp' => sub { + plan tests => 4; + + my $cmd_pps_template = { + version => '2.0', + description => 'subst: cmd postprocessing (string)', + nodes => [ + { + id => 'node0', + type => 'EXEC', + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => 'echo simple string', + }, + ], + }; + + my $cmd_ppa_template = { + version => '2.0', + description => 'subst: cmd postprocessing (array)', + subst_params => [ + { id => 'nest', default => [ 'makers', 'of', {subst => 'product'}, ] } , + { id => 'product', default => [ 'the', 'very', 'best', 'chocolate', ] } , + { id => 'list', default => [ 'one', 'two', 'three', ], } , + ], + nodes => [ + { + id => 'node0', + type => 'EXEC', + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ + 'echo', + 'wheels within wheels', + {subst => 'nest'}, + {subst => 'list'} + ], + }, + ], + }; + + my $template = $tdir.q[/10-vtfp-cmd_pps_template.json]; + my $template_contents = to_json($cmd_pps_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + id => 'node0', + type => 'EXEC', + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => 'echo simple string', + }, + ], + edges => [], + }; + + is_deeply ($vtfp_results, $expected_result, 'subst locality'); + + # now cmd as a nest of arrays that should resolve to an array of strings + $template = $tdir.q[/10-vtfp-cmd_ppa_template.json]; + $template_contents = to_json($cmd_ppa_template); + write_file($template, $template_contents); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + id => 'node0', + type => 'EXEC', + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ + 'echo', + 'wheels within wheels', + 'makers', + 'of', + 'the', + 'very', + 'best', + 'chocolate', + 'one', + 'two', + 'three', + ], + }, + ], + edges => [], + }; + + is_deeply ($vtfp_results, $expected_result, 'subst locality'); +}; + +1; diff --git a/t/10-vtfp-vtfile.mod.t b/t/10-vtfp-vtfile.mod.t new file mode 100644 index 000000000..a1be74ebb --- /dev/null +++ b/t/10-vtfp-vtfile.mod.t @@ -0,0 +1,412 @@ +use strict; +use warnings; +use Carp; +use Test::More tests => 4; +use Test::Cmd; +use File::Slurp; +use Perl6::Slurp; +use JSON; +use File::Temp qw(tempdir); +use Cwd; + +my $tdir = tempdir(CLEANUP => 1); + +my $odir = getcwd(); +my $test = Test::Cmd->new( prog => $odir.'/bin/vtfp.pl', workdir => q()); +ok($test, 'made test object'); + +# Test VTFILE processing by vtfp + +# basic functions +subtest 'basic_checks' => sub { + plan tests => 2; + + my $basic_container = { + description => 'basic template containing a VTFILE node', + version => '1.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => [ 'echo', 'aeronautics'] + }, + { + id => 'v1', + type => 'VTFILE', + node_prefix => 'vtf0_', + name => "$tdir/10-vtfp-vtfile_vtf0.json" + } + ], + edges => [ + { id => 'e1', from => 'n1', to => 'v1'} + ] + }; + + my $vtf0 = { + description => 'basic VTFILE', + version => '1.0', + subgraph_io => { + ports => { + inputs => { + _stdin_ => 'vowelrot' + } + } + }, + nodes => [ + { + id => 'vowelrot', + type => 'EXEC', + cmd => [ 'tr', 'aeiou', 'eioua' ] + } + ] + }; + + my $template = $tdir.q[/10-vtfp-vtfile_basic.json]; + my $template_contents = to_json($basic_container); + write_file($template, $template_contents); + + my $vtfile = $tdir.q[/10-vtfp-vtfile_vtf0.json]; + my $vtfile_contents = to_json($vtf0); + write_file($vtfile, $vtfile_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '1.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo', 'aeronautics'] + }, + { + id => 'vtf0_vowelrot', + type => 'EXEC', + cmd => [ 'tr', 'aeiou', 'eioua' ] + } + ], + edges=> [ + { id => 'e1', from => 'n1', to => 'vtf0_vowelrot'} + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'basic check'); +}; + +subtest 'multilevel_vtf' => sub { + plan tests => 4; + + my $basic_container = { + description => 'basic template containing a VTFILE node', + version => '1.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => [ 'echo', 'aeronautics'] + }, + { + id => 'v1', + type => 'VTFILE', + node_prefix => 'vtf1_', + name => { subst => 'vtfname', ifnull => "$tdir/10-vtfp-vtfile_vtf1.json" } + } + ], + edges => [ + { id => 'e1', from => 'n1', to => 'v1'} + ] + }; + + my $vtf1 = { + description => 'unary', + version => '1.0', + subgraph_io => { + ports => { + inputs => { + _stdin_ => 'rev', + } + } + }, + nodes => [ + { + id => 'rev', + type => 'EXEC', + cmd => [ 'rev' ] + }, + { + id => 'file', + type => 'OUTFILE', + name => { subst_constructor => { vals => [ 'tmp.', {subst => 'ext', ifnull => 'txt', }], postproc => { op => 'concat', pad => ''} }, } + } + ], + edges => [ + { id => 'e2', from => 'rev', to => 'file'} + ] + }; + + my $vtf2 = { + description => 'binary', + version => '1.0', + subgraph_io => { + ports => { + inputs => { + _stdin_ => 'tee', + } + } + }, + nodes => [ + { + id => 'tee', + type => 'EXEC', + cmd => [ 'tee', '__A_OUT__', '__B_OUT__' ] + }, + { + id => 'aout', + type => 'VTFILE', + name => "$tdir/10-vtfp-vtfile_vtf1.json", + node_prefix => 'aout_', + subst_map => { ext =>'xxx' }, + }, + { + id => 'bout', + type => 'VTFILE', + name => "$tdir/10-vtfp-vtfile_vtf1.json", + node_prefix => 'bout_', + subst_map => { ext => 'yyy' }, + }, + ], + edges => [ + { id => 'e3', from => 'tee:__A_OUT__', to => 'aout'}, + { id => 'e4', from => 'tee:__B_OUT__', to => 'bout'}, + ] + }; + + my $template = $tdir.q[/10-vtfp-vtfile_multilevel0.json]; + my $template_contents = to_json($basic_container); + write_file($template, $template_contents); + + my $vtfile1 = $tdir.q[/10-vtfp-vtfile_vtf1.json]; + my $vtfile_contents = to_json($vtf1); + write_file($vtfile1, $vtfile_contents); + + my $vtfile2 = $tdir.q[/10-vtfp-vtfile_vtf2.json]; + $vtfile_contents = to_json($vtf2); + write_file($vtfile2, $vtfile_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '1.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo', 'aeronautics'] + }, + { + id => 'vtf1_rev', + type => 'EXEC', + cmd => [ 'rev' ] + }, + { + id => 'vtf1_file', + type => 'OUTFILE', + name => 'tmp.txt' + } + ], + edges=> [ + { id => 'e1', from => 'n1', to => 'vtf1_rev'}, + { id => 'e2', from => 'vtf1_rev', to => 'vtf1_file'} + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'multilevel VTFILE nodes - first just one'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys vtfname -vals $vtfile2 -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '1.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo', 'aeronautics'] + }, + { + id => 'vtf1_tee', + type => 'EXEC', + cmd => [ 'tee', '__A_OUT__', '__B_OUT__' ] + }, + { + id => 'aout_rev', + type => 'EXEC', + cmd => [ 'rev' ] + }, + { + id => 'aout_file', + type => 'OUTFILE', + name => 'tmp.xxx' + }, + { + id => 'bout_rev', + type => 'EXEC', + cmd => [ 'rev' ] + }, + { + id => 'bout_file', + type => 'OUTFILE', + name => 'tmp.yyy' + } + ], + edges=> [ + { id => 'e1', from => 'n1', to => 'vtf1_tee'}, + { id => 'e3', from => 'vtf1_tee:__A_OUT__', to => 'aout_rev'}, + { id => 'e4', from => 'vtf1_tee:__B_OUT__', to => 'bout_rev'}, + { id => 'e2', from => 'aout_rev', to => 'aout_file'}, + { id => 'e2', from => 'bout_rev', to => 'bout_file'} + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'multilevel VTFILE nodes - two level (split)'); +}; + +subtest 'multilevel_local_param_reeval' => sub { + plan tests => 2; + + my $basic_container = { + description => 'top template containing a VTFILE node', + version => '1.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => [ 'echo', 'aeronautics'] + }, + { + id => 'v1', + type => 'VTFILE', + node_prefix => 'vtf11_', + name => "$tdir/10-vtfp-vtfile_vtf11.json", + subst_map => { component => 'xy' } + } + ], + edges => [ + { id => 'e1', from => 'n1', to => 'v1'} + ] + }; + + my $vtf11 = { + description => 'mid', + version => '1.0', + subgraph_io => { + ports => { + inputs => { + _stdin_ => 'tee', + } + } + }, + subst_params => [ + { id => 'ext', subst_constructor => {vals => [ 'w', {subst => 'component'}, 'z' ], postproc => { op => 'concat', pad => ''}} } + ], + nodes => [ + { + id => 'tee', + type => 'EXEC', + cmd => [ 'tee', '__A_OUT__', '__B_OUT__' ] + }, + { + id => 'file', + type => 'OUTFILE', + name => { subst_constructor => { vals => [ 'tmp.', {subst => 'ext', ifnull => 'tat', }], postproc => { op => 'concat', pad => ''} }, } + }, + { + id => 'vfile', + type => 'VTFILE', + node_prefix => 'vtf12_', + name => "$tdir/10-vtfp-vtfile_vtf12.json", + subst_map => { component => 'ee' } + }, + ], + edges => [ + { id => 'e2', from => 'tee:__A_OUT__', to => 'file'}, + { id => 'e3', from => 'tee:__B_OUT__', to => 'vfile'}, + ] + }; + + my $vtf12 = { + description => 'bottom', + comment => 'the value of param ext should not be inherited from the cache of the parent, since the passed component value should force local reevaluation', + version => '1.0', + subgraph_io => { + ports => { + inputs => { + _stdin_ => 'vfile', + } + } + }, + nodes => [ + { + id => 'vfile', + type => 'OUTFILE', + name => { subst_constructor => { vals => [ 'tmp.', {subst => 'ext'} ], postproc => { op => 'concat', pad => ''} }, } + }, + ] + }; + + my $template = $tdir.q[/10-vtfp-vtfile_multilevel1.json]; + my $template_contents = to_json($basic_container); + write_file($template, $template_contents); + + my $vtfile11 = $tdir.q[/10-vtfp-vtfile_vtf11.json]; + my $vtfile_contents = to_json($vtf11); + write_file($vtfile11, $vtfile_contents); + + my $vtfile12 = $tdir.q[/10-vtfp-vtfile_vtf12.json]; + $vtfile_contents = to_json($vtf12); + write_file($vtfile12, $vtfile_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '1.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo', 'aeronautics'] + }, + { + id => 'vtf11_tee', + type => 'EXEC', + cmd => [ 'tee', '__A_OUT__', '__B_OUT__' ] + }, + { + id => 'vtf11_file', + type => 'OUTFILE', + name => 'tmp.wxyz', + }, + { + id => 'vtf12_vfile', + type => 'OUTFILE', + name => 'tmp.weez', + }, + ], + edges=> [ + { id => 'e1', from => 'n1', to => 'vtf11_tee'}, + { id => 'e2', from => 'vtf11_tee:__A_OUT__', to => 'vtf11_file'}, + { id => 'e3', from => 'vtf11_tee:__B_OUT__', to => 'vtf12_vfile'} + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'multilevel local param reeval'); +}; + +1; diff --git a/t/10-vtfp-vtfile.t b/t/10-vtfp-vtfile.t index ee4b23b6f..a1be74ebb 100644 --- a/t/10-vtfp-vtfile.t +++ b/t/10-vtfp-vtfile.t @@ -48,9 +48,7 @@ subtest 'basic_checks' => sub { subgraph_io => { ports => { inputs => { - _stdin_ => 'vowelrot', - target_seqchksum => 'merge_output_seqchksum:__TARGET_CHKSUM_IN__', - phix_seqchksum => 'merge_output_seqchksum:__PHIX_CHKSUM_IN__' + _stdin_ => 'vowelrot' } } }, @@ -76,6 +74,7 @@ subtest 'basic_checks' => sub { my $vtfp_results = from_json($test->stdout); my $expected_result = { + version => '1.0', nodes => [ { id => 'n1', @@ -201,6 +200,7 @@ subtest 'multilevel_vtf' => sub { my $vtfp_results = from_json($test->stdout); my $expected_result = { + version => '1.0', nodes => [ { id => 'n1', @@ -231,6 +231,7 @@ subtest 'multilevel_vtf' => sub { $vtfp_results = from_json($test->stdout); $expected_result = { + version => '1.0', nodes => [ { id => 'n1', @@ -375,6 +376,7 @@ subtest 'multilevel_local_param_reeval' => sub { my $vtfp_results = from_json($test->stdout); my $expected_result = { + version => '1.0', nodes => [ { id => 'n1', diff --git a/t/10-vtfp-vtfile_v2.t b/t/10-vtfp-vtfile_v2.t new file mode 100644 index 000000000..a60e154f8 --- /dev/null +++ b/t/10-vtfp-vtfile_v2.t @@ -0,0 +1,412 @@ +use strict; +use warnings; +use Carp; +use Test::More tests => 4; +use Test::Cmd; +use File::Slurp; +use Perl6::Slurp; +use JSON; +use File::Temp qw(tempdir); +use Cwd; + +my $tdir = tempdir(CLEANUP => 1); + +my $odir = getcwd(); +my $test = Test::Cmd->new( prog => $odir.'/bin/vtfp.pl', workdir => q()); +ok($test, 'made test object'); + +# Test VTFILE processing by vtfp + +# basic functions +subtest 'basic_checks' => sub { + plan tests => 2; + + my $basic_container = { + description => 'basic template containing a VTFILE node', + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => [ 'echo', 'aeronautics'] + }, + { + id => 'v1', + type => 'VTFILE', + node_prefix => 'vtf0_', + name => "$tdir/10-vtfp-vtfile_vtf0.json" + } + ], + edges => [ + { id => 'e1', from => 'n1', to => 'v1'} + ] + }; + + my $vtf0 = { + description => 'basic VTFILE', + version => '2.0', + subgraph_io => { + ports => { + inputs => { + _stdin_ => 'vowelrot' + } + } + }, + nodes => [ + { + id => 'vowelrot', + type => 'EXEC', + cmd => [ 'tr', 'aeiou', 'eioua' ] + } + ] + }; + + my $template = $tdir.q[/10-vtfp-vtfile_basic.json]; + my $template_contents = to_json($basic_container); + write_file($template, $template_contents); + + my $vtfile = $tdir.q[/10-vtfp-vtfile_vtf0.json]; + my $vtfile_contents = to_json($vtf0); + write_file($vtfile, $vtfile_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo', 'aeronautics'] + }, + { + id => 'vtf0_vowelrot', + type => 'EXEC', + cmd => [ 'tr', 'aeiou', 'eioua' ] + } + ], + edges=> [ + { id => 'e1', from => 'n1', to => 'vtf0_vowelrot'} + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'basic check'); +}; + +subtest 'multilevel_vtf' => sub { + plan tests => 4; + + my $basic_container = { + description => 'basic template containing a VTFILE node', + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => [ 'echo', 'aeronautics'] + }, + { + id => 'v1', + type => 'VTFILE', + node_prefix => 'vtf1_', + name => { subst => 'vtfname', ifnull => "$tdir/10-vtfp-vtfile_vtf1.json" } + } + ], + edges => [ + { id => 'e1', from => 'n1', to => 'v1'} + ] + }; + + my $vtf1 = { + description => 'unary', + version => '2.0', + subgraph_io => { + ports => { + inputs => { + _stdin_ => 'rev', + } + } + }, + nodes => [ + { + id => 'rev', + type => 'EXEC', + cmd => [ 'rev' ] + }, + { + id => 'file', + type => 'OUTFILE', + name => { subst_constructor => { vals => [ 'tmp.', {subst => 'ext', ifnull => 'txt', }], postproc => { op => 'concat', pad => ''} }, } + } + ], + edges => [ + { id => 'e2', from => 'rev', to => 'file'} + ] + }; + + my $vtf2 = { + description => 'binary', + version => '2.0', + subgraph_io => { + ports => { + inputs => { + _stdin_ => 'tee', + } + } + }, + nodes => [ + { + id => 'tee', + type => 'EXEC', + cmd => [ 'tee', {'port' => 'a'} , {'port' => 'b'}, ] + }, + { + id => 'aout', + type => 'VTFILE', + name => "$tdir/10-vtfp-vtfile_vtf1.json", + node_prefix => 'aout_', + subst_map => { ext =>'xxx' }, + }, + { + id => 'bout', + type => 'VTFILE', + name => "$tdir/10-vtfp-vtfile_vtf1.json", + node_prefix => 'bout_', + subst_map => { ext => 'yyy' }, + }, + ], + edges => [ + { id => 'e3', from => 'tee:a', to => 'aout'}, + { id => 'e4', from => 'tee:b', to => 'bout'}, + ] + }; + + my $template = $tdir.q[/10-vtfp-vtfile_multilevel0.json]; + my $template_contents = to_json($basic_container); + write_file($template, $template_contents); + + my $vtfile1 = $tdir.q[/10-vtfp-vtfile_vtf1.json]; + my $vtfile_contents = to_json($vtf1); + write_file($vtfile1, $vtfile_contents); + + my $vtfile2 = $tdir.q[/10-vtfp-vtfile_vtf2.json]; + $vtfile_contents = to_json($vtf2); + write_file($vtfile2, $vtfile_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo', 'aeronautics'] + }, + { + id => 'vtf1_rev', + type => 'EXEC', + cmd => [ 'rev' ] + }, + { + id => 'vtf1_file', + type => 'OUTFILE', + name => 'tmp.txt' + } + ], + edges=> [ + { id => 'e1', from => 'n1', to => 'vtf1_rev'}, + { id => 'e2', from => 'vtf1_rev', to => 'vtf1_file'} + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'multilevel VTFILE nodes - first just one'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys vtfname -vals $vtfile2 -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo', 'aeronautics'] + }, + { + id => 'vtf1_tee', + type => 'EXEC', + cmd => [ 'tee', {'port' => 'a'} , {'port' => 'b'}, ] + }, + { + id => 'aout_rev', + type => 'EXEC', + cmd => [ 'rev' ] + }, + { + id => 'aout_file', + type => 'OUTFILE', + name => 'tmp.xxx' + }, + { + id => 'bout_rev', + type => 'EXEC', + cmd => [ 'rev' ] + }, + { + id => 'bout_file', + type => 'OUTFILE', + name => 'tmp.yyy' + } + ], + edges=> [ + { id => 'e1', from => 'n1', to => 'vtf1_tee'}, + { id => 'e3', from => 'vtf1_tee:a', to => 'aout_rev'}, + { id => 'e4', from => 'vtf1_tee:b', to => 'bout_rev'}, + { id => 'e2', from => 'aout_rev', to => 'aout_file'}, + { id => 'e2', from => 'bout_rev', to => 'bout_file'} + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'multilevel VTFILE nodes - two level (split)'); +}; + +subtest 'multilevel_local_param_reeval' => sub { + plan tests => 2; + + my $basic_container = { + description => 'top template containing a VTFILE node', + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => [ 'echo', 'aeronautics'] + }, + { + id => 'v1', + type => 'VTFILE', + node_prefix => 'vtf11_', + name => "$tdir/10-vtfp-vtfile_vtf11.json", + subst_map => { component => 'xy' } + } + ], + edges => [ + { id => 'e1', from => 'n1', to => 'v1'} + ] + }; + + my $vtf11 = { + description => 'mid', + version => '2.0', + subgraph_io => { + ports => { + inputs => { + _stdin_ => 'tee', + } + } + }, + subst_params => [ + { id => 'ext', subst_constructor => {vals => [ 'w', {subst => 'component'}, 'z' ], postproc => { op => 'concat', pad => ''}} } + ], + nodes => [ + { + id => 'tee', + type => 'EXEC', + cmd => [ 'tee', '__A_OUT__', '__B_OUT__' ] + }, + { + id => 'file', + type => 'OUTFILE', + name => { subst_constructor => { vals => [ 'tmp.', {subst => 'ext', ifnull => 'tat', }], postproc => { op => 'concat', pad => ''} }, } + }, + { + id => 'vfile', + type => 'VTFILE', + node_prefix => 'vtf12_', + name => "$tdir/10-vtfp-vtfile_vtf12.json", + subst_map => { component => 'ee' } + }, + ], + edges => [ + { id => 'e2', from => 'tee:__A_OUT__', to => 'file'}, + { id => 'e3', from => 'tee:__B_OUT__', to => 'vfile'}, + ] + }; + + my $vtf12 = { + description => 'bottom', + comment => 'the value of param ext should not be inherited from the cache of the parent, since the passed component value should force local reevaluation', + version => '2.0', + subgraph_io => { + ports => { + inputs => { + _stdin_ => 'vfile', + } + } + }, + nodes => [ + { + id => 'vfile', + type => 'OUTFILE', + name => { subst_constructor => { vals => [ 'tmp.', {subst => 'ext'} ], postproc => { op => 'concat', pad => ''} }, } + }, + ] + }; + + my $template = $tdir.q[/10-vtfp-vtfile_multilevel1.json]; + my $template_contents = to_json($basic_container); + write_file($template, $template_contents); + + my $vtfile11 = $tdir.q[/10-vtfp-vtfile_vtf11.json]; + my $vtfile_contents = to_json($vtf11); + write_file($vtfile11, $vtfile_contents); + + my $vtfile12 = $tdir.q[/10-vtfp-vtfile_vtf12.json]; + $vtfile_contents = to_json($vtf12); + write_file($vtfile12, $vtfile_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo', 'aeronautics'] + }, + { + id => 'vtf11_tee', + type => 'EXEC', + cmd => [ 'tee', '__A_OUT__', '__B_OUT__' ] + }, + { + id => 'vtf11_file', + type => 'OUTFILE', + name => 'tmp.wxyz', + }, + { + id => 'vtf12_vfile', + type => 'OUTFILE', + name => 'tmp.weez', + }, + ], + edges=> [ + { id => 'e1', from => 'n1', to => 'vtf11_tee'}, + { id => 'e2', from => 'vtf11_tee:__A_OUT__', to => 'vtf11_file'}, + { id => 'e3', from => 'vtf11_tee:__B_OUT__', to => 'vtf12_vfile'} + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'multilevel local param reeval'); +}; + +1; diff --git a/t/50-viv.t b/t/50-viv.t index d06c75a2c..52bcf4672 100644 --- a/t/50-viv.t +++ b/t/50-viv.t @@ -1,6 +1,6 @@ use strict; use warnings; -use Test::More tests => 11; +use Test::More tests => 21; use Test::Cmd; use Cwd; @@ -9,15 +9,21 @@ my$odir=getcwd(); my $test = Test::Cmd->new( prog => $odir.'/bin/viv.pl', workdir => q()); #, match_sub => sub{my($ret,$exp)=@_; return 0 ; $ret=~m/\Q$exp\E/smx} ); ok($test, 'made test object'); foreach( - ['50-viv_failing_io_pipeline0.vtf', 255, 'Node d has poorly described port __IN1__'], - ['50-viv_failing_io_pipeline1.vtf', 255, 'Node d has no port __IN_N2__'], - ['50-viv_failing_io_pipeline2.vtf', 255, 'Node d port __OUT_2__ connected as "to"'], - ['50-viv_failing_pipeline.vtf', 10, 'Exiting due to abnormal return from child n2'], - ['50-viv_pipeline.vtf', 0, '(viv) - Done'] + ['50-viv_failing_io_pipeline0.vtf', 255, 'ERROR: to port d:__IN1__ referenced in edge (n2:__stdout__ => d:__IN1__), but no corresponding port found in nodes'], + ['50-viv_failing_io_pipeline1.vtf', 255, 'ERROR: to port d:__IN_N2__ referenced in edge (m:__stdout__ => d:__IN_N2__), but no corresponding port found in nodes'], + ['50-viv_failing_io_pipeline2.vtf', 255, 'ERROR: to port to_id:__OUT_2__ refers to output port in node'], + ['50-viv_failing_pipeline.vtf', 10, 'Exiting due to abnormal status return from child n2'], + ['50-viv_pipeline.vtf', 0, '(viv) - Done'], + ['50-viv_failing_io_pipeline0.v2.vtf', 255, 'ERROR: to port to_id:__IN1__ refers to output port in node'], + ['50-viv_failing_io_pipeline1.v2.vtf', 255, 'ERROR: to port d:__IN_N2__ referenced in edge (m:__stdout__ => d:__IN_N2__), but no corresponding port found in nodes'], + ['50-viv_failing_io_pipeline2.v2.vtf', 255, 'ERROR: to port to_id:__OUT_2__ refers to output port in node'], + ['50-viv_failing_pipeline.v2.vtf', 10, 'Exiting due to abnormal status return from child n2'], + ['50-viv_pipeline.v2.vtf', 0, '(viv) - Done'], ){ my($vtf, $estatus, $eerror)=@$_; my$exit_status = $test->run(chdir => $test->curdir, args => "-s -x $odir/t/data/$vtf"); cmp_ok($exit_status>>8, q(==), $estatus, "expected exit status of $estatus for $vtf"); +#print ">>>> ", $test->stderr; like($test->stderr,qr(\Q$eerror\E)smx, "expected err info for $vtf"); } diff --git a/t/data/10-vtfp-array_expansion.v2.json b/t/data/10-vtfp-array_expansion.v2.json new file mode 100644 index 000000000..3030fc5d8 --- /dev/null +++ b/t/data/10-vtfp-array_expansion.v2.json @@ -0,0 +1,19 @@ +{ +"description":"Test expansion of parameters, specifically cases where there are multiple instances of multi-valued parameters (arrays)", +"version":"2.0", +"subst_params":[ + { "id": "p1", "subst_constructor":{ "vals":[ "1A", "1B" ] } }, + { "id": "p2", "subst_constructor":{ "vals":[ "2A" ] } }, + { "id": "p3", "subst_constructor":{ "vals":[ "3A", "3B", "3C", "3D" ] } }, + { "id": "p4", "default":"4A"}, + { "id": "p5", "subst_constructor":{ "vals":[ "5A", "5B" ] } } +], +"nodes":[ + { + "id":"n1", + "type":"EXEC", + "cmd":[ "echo", {"subst":"p1"}, {"subst":"p2"}, {"subst":"p3"}, {"subst":"p4"}, {"subst":"p5"} ] + } +] +} + diff --git a/t/data/50-viv_failing_io_pipeline0.v2.vtf b/t/data/50-viv_failing_io_pipeline0.v2.vtf new file mode 100644 index 000000000..bd3bc7062 --- /dev/null +++ b/t/data/50-viv_failing_io_pipeline0.v2.vtf @@ -0,0 +1,41 @@ +{ + "description": "minimal failing test pipeline - fails as I/O port names are poorly named", + "version":2.0, + "nodes":[ + { "id": "n1", + "type": "EXEC", + "cmd": "echo stuff", + "use_STDIN": 0, + "use_STDOUT": 1 + }, + { "id": "n2", + "type": "EXEC", + "cmd": ["cat"], + "use_STDIN": 1, + "use_STDOUT": 1 + }, + { "id": "m", + "type": "EXEC", + "cmd": ["echo", "stuff"], + "use_STDIN": false, + "use_STDOUT": true + }, + { "id": "d", + "type": "EXEC", + "cmd": [ "diff", {"port":"__IN1__", "direction":"out"}, {"port":"__IN2__"} ], + "use_STDIN": 0, + "use_STDOUT": 0 + } + ], + "edges":[ + { "from": "n1", + "to": "n2" + }, + { "from": "n2", + "to": "d:__IN1__" + }, + { "from": "m", + "to": "d:__IN2__" + } + ] +} diff --git a/t/data/50-viv_failing_io_pipeline0.vtf b/t/data/50-viv_failing_io_pipeline0.vtf index eec000ba8..e96e9b858 100644 --- a/t/data/50-viv_failing_io_pipeline0.vtf +++ b/t/data/50-viv_failing_io_pipeline0.vtf @@ -1,5 +1,6 @@ { "description": "minimal failing test pipeline - fails as I/O port names are poorly named", + "version": "1.0", "nodes":[ { "id": "n1", "type": "EXEC", diff --git a/t/data/50-viv_failing_io_pipeline1.v2.vtf b/t/data/50-viv_failing_io_pipeline1.v2.vtf new file mode 100644 index 000000000..0785e36f5 --- /dev/null +++ b/t/data/50-viv_failing_io_pipeline1.v2.vtf @@ -0,0 +1,41 @@ +{ + "description": "minimal failing test pipeline. Port name in edge does not match one in node definition", + "version":2.0, + "nodes":[ + { "id": "n1", + "type": "EXEC", + "cmd": "echo stuff", + "use_STDIN": 0, + "use_STDOUT": 1 + }, + { "id": "n2", + "type": "EXEC", + "cmd": ["cat"], + "use_STDIN": 1, + "use_STDOUT": 1 + }, + { "id": "m", + "type": "EXEC", + "cmd": ["echo", "stuff"], + "use_STDIN": false, + "use_STDOUT": true + }, + { "id": "d", + "type": "EXEC", + "cmd": ["diff", {"port":"__IN_1__"}, {"port":"__IN_2__"}], + "use_STDIN": 0, + "use_STDOUT": 0 + } + ], + "edges":[ + { "from": "n1", + "to": "n2" + }, + { "from": "n2", + "to": "d:__IN_1__" + }, + { "from": "m", + "to": "d:__IN_N2__" + } + ] +} diff --git a/t/data/50-viv_failing_io_pipeline1.vtf b/t/data/50-viv_failing_io_pipeline1.vtf index 004de08d8..81b0b2f4d 100644 --- a/t/data/50-viv_failing_io_pipeline1.vtf +++ b/t/data/50-viv_failing_io_pipeline1.vtf @@ -1,5 +1,6 @@ { "description": "minimal failing test pipeline. Port name in edge does not match one in node definition", + "version":1.0, "nodes":[ { "id": "n1", "type": "EXEC", diff --git a/t/data/50-viv_failing_io_pipeline2.v2.vtf b/t/data/50-viv_failing_io_pipeline2.v2.vtf new file mode 100644 index 000000000..5e8c32dd9 --- /dev/null +++ b/t/data/50-viv_failing_io_pipeline2.v2.vtf @@ -0,0 +1,41 @@ +{ + "description": "minimal failing test pipeline. Port names in edges match the node definitions, but are of the wrong type", + "version":2.0, + "nodes":[ + { "id": "n1", + "type": "EXEC", + "cmd": "echo stuff", + "use_STDIN": 0, + "use_STDOUT": 1 + }, + { "id": "n2", + "type": "EXEC", + "cmd": ["cat"], + "use_STDIN": 1, + "use_STDOUT": 1 + }, + { "id": "m", + "type": "EXEC", + "cmd": ["echo", "stuff"], + "use_STDIN": false, + "use_STDOUT": true + }, + { "id": "d", + "type": "EXEC", + "cmd": ["diff", {"port":"__IN_1__"}, {"port":"__OUT_2__", "direction":"out"}], + "use_STDIN": 0, + "use_STDOUT": 0 + } + ], + "edges":[ + { "from": "n1", + "to": "n2" + }, + { "from": "n2", + "to": "d:__IN_1__" + }, + { "from": "m", + "to": "d:__OUT_2__" + } + ] +} diff --git a/t/data/50-viv_failing_io_pipeline2.vtf b/t/data/50-viv_failing_io_pipeline2.vtf index 91b9b683f..9e3e0c8c1 100644 --- a/t/data/50-viv_failing_io_pipeline2.vtf +++ b/t/data/50-viv_failing_io_pipeline2.vtf @@ -1,5 +1,6 @@ { "description": "minimal failing test pipeline. Port names in edges match the node definitions, but are of the wrong type", + "version":1.0, "nodes":[ { "id": "n1", "type": "EXEC", diff --git a/t/data/50-viv_failing_pipeline.v2.vtf b/t/data/50-viv_failing_pipeline.v2.vtf new file mode 100644 index 000000000..7340cbcf3 --- /dev/null +++ b/t/data/50-viv_failing_pipeline.v2.vtf @@ -0,0 +1,38 @@ +{ + "description": "minimal failing test pipeline. Fails because of false in cmd", + "version":2.0, + "nodes":[ + { "id": "n1", + "type": "EXEC", + "use_STDOUT":true, + "cmd": "echo stuff; sleep 1 && cat" + }, + { "id": "n2", + "type": "EXEC", + "use_STDIN":true, + "use_STDOUT":true, + "ocmd": "cat; false", + "cmd": "head -1; false" + }, + { "id": "m", + "type": "EXEC", + "use_STDOUT":true, + "cmd": ["echo", "stuff"] + }, + { "id": "d", + "type": "EXEC", + "cmd": ["diff", {"port":"__IN_1__"}, {"port":"__IN_2__"}] + } + ], + "edges":[ + { "from": "n1", + "to": "n2" + }, + { "from": "n2", + "to": "d:__IN_1__" + }, + { "from": "m", + "to": "d:__IN_2__" + } + ] +} diff --git a/t/data/50-viv_failing_pipeline.vtf b/t/data/50-viv_failing_pipeline.vtf index e64fb0c64..20c20bb18 100644 --- a/t/data/50-viv_failing_pipeline.vtf +++ b/t/data/50-viv_failing_pipeline.vtf @@ -1,17 +1,22 @@ { "description": "minimal failing test pipeline. Fails because of false in cmd", + "version":1.0, "nodes":[ { "id": "n1", "type": "EXEC", + "use_STDOUT":true, "cmd": "echo stuff; sleep 1 && cat" }, { "id": "n2", "type": "EXEC", + "use_STDIN":true, + "use_STDOUT":true, "ocmd": "cat; false", "cmd": "head -1; false" }, { "id": "m", "type": "EXEC", + "use_STDOUT":true, "cmd": ["echo", "stuff"] }, { "id": "d", diff --git a/t/data/50-viv_pipeline.v2.vtf b/t/data/50-viv_pipeline.v2.vtf new file mode 100644 index 000000000..1e44e665a --- /dev/null +++ b/t/data/50-viv_pipeline.v2.vtf @@ -0,0 +1,41 @@ +{ + "description": "minimal test pipeline", + "version":2.0, + "nodes":[ + { "id": "n1", + "type": "EXEC", + "cmd": "echo stuff", + "use_STDIN": 0, + "use_STDOUT": 1 + }, + { "id": "n2", + "type": "EXEC", + "cmd": ["cat"], + "use_STDIN": 1, + "use_STDOUT": 1 + }, + { "id": "m", + "type": "EXEC", + "cmd": ["echo", "stuff"], + "use_STDIN": false, + "use_STDOUT": true + }, + { "id": "d", + "type": "EXEC", + "cmd": ["diff", {"port":"__IN_1__"}, {"port":"__IN_2__"}], + "use_STDIN": 0, + "use_STDOUT": 0 + } + ], + "edges":[ + { "from": "n1", + "to": "n2" + }, + { "from": "n2", + "to": "d:__IN_1__" + }, + { "from": "m", + "to": "d:__IN_2__" + } + ] +} diff --git a/t/data/50-viv_pipeline.vtf b/t/data/50-viv_pipeline.vtf index 631f2835d..937453a05 100644 --- a/t/data/50-viv_pipeline.vtf +++ b/t/data/50-viv_pipeline.vtf @@ -1,5 +1,6 @@ { "description": "minimal test pipeline", + "version":1.0, "nodes":[ { "id": "n1", "type": "EXEC", @@ -21,7 +22,7 @@ }, { "id": "d", "type": "EXEC", - "cmd": "diff __IN_1__ __IN_2__", + "cmd": ["diff", "__IN_1__", "__IN_2__"], "use_STDIN": 0, "use_STDOUT": 0 } From 350a3bd08624a54cc27a8440f9595122201faf66 Mon Sep 17 00:00:00 2001 From: "Ruben E. Bautista" Date: Tue, 21 Feb 2017 15:23:45 +0000 Subject: [PATCH 03/20] STAR Aligner alignment template - initial commit - Works fine but the format of some outputs can be improved or they may not be necessary in the first place --- data/vtlib/star_alignment.json | 290 +++++++++++++++++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 data/vtlib/star_alignment.json diff --git a/data/vtlib/star_alignment.json b/data/vtlib/star_alignment.json new file mode 100644 index 000000000..945d16bf9 --- /dev/null +++ b/data/vtlib/star_alignment.json @@ -0,0 +1,290 @@ +{ +"description":"run star to to align input bam to supplied reference genome", +"version":"1.0", +"subgraph_io":{ + "ports":{ + "inputs":{ + "_stdin_":"bamtofastq", + "reference":"star:__REFERENCE_GENOME_IN__" + }, + "outputs":{ + "_stdout_":"star" + } + } +}, +"subst_params":[ + { + "id": "basic_pipeline_params", + "type":"SPFILE", + "name":{"subst":"basic_pipeline_params_file"}, + "required": "no", + "comment":"this will expand to a set of subst_param elements" + }, + { + "id":"fastq1_name", + "required":"no", + "default":"intfile_1.fq.gz", + "subst_constructor":{ + "vals":[ "intfile_1_", {"subst":"rpt"}, ".fq.gz" ], + "postproc":{"op":"concat", "pad":""} + } + }, + { + "id":"fastq1", + "required":"yes", + "subst_constructor":{ + "vals":[ {"subst":"tmpdir"}, "/", {"subst":"fastq1_name"} ], + "postproc":{"op":"concat", "pad":""} + } + }, + { + "id":"fastq2_name", + "required":"no", + "default":"intfile_2.fq.gz", + "subst_constructor":{ + "vals":[ "intfile_2_", {"subst":"rpt"}, ".fq.gz" ], + "postproc":{"op":"concat", "pad":""} + } + }, + { + "id":"fastq2", + "required":"yes", + "subst_constructor":{ + "vals":[ {"subst":"tmpdir"}, "/", {"subst":"fastq2_name"} ], + "postproc":{"op":"concat", "pad":""} + } + }, + { + "id":"star_dir","required":"no","default":"star_out" + }, + { + "id":"star_out", + "required":"no", + "default": "star_out", + "subst_constructor":{ + "vals":[ {"subst":"star_dir"}, "_", {"subst":"rpt"}, "/" ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"transcriptome_subpath","required":"no" + }, + { + "id":"sjdb_annotation_val", + "subst_constructor":{ + "vals":[ {"subst":"reposdir"}, "/transcriptomes/", {"subst":"transcriptome_subpath"} ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"sjdb_annotation_flag", + "required":"no", + "subst_constructor":{ + "vals":[ "--sjdbGTFfile", {"subst":"sjdb_annotation_val"} ], + "postproc":{"op":"concat","pad":" "} + } + }, + { + "id":"aligner_numthreads_flag", + "required":"no", + "subst_constructor":{ + "vals":[ "--runThreadN", {"subst":"aligner_numthreads"} ], + "postproc":{"op":"concat","pad":" "} + } + }, + { + "id":"sjdb_overhang_val", + "required":"no", + "default":"99" + }, + { + "id":"sjdb_overhang_flag", + "required":"no", + "subst_constructor":{ + "vals":[ "--sjdbOverhang", {"subst":"sjdb_overhang_val"} ], + "postproc":{"op":"concat","pad":" "} + } + }, + { + "id":"junctions_tab", + "required":"no", + "default":"star_out/SJ.out.tab", + "subst_constructor":{ + "vals":[ {"subst":"star_dir"}, "_", {"subst":"rpt"}, "/SJ.out.tab" ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"totranscriptome_bam", + "required":"no", + "default":"star_out/Aligned.toTranscriptome.out.bam", + "subst_constructor":{ + "vals":[ {"subst":"star_dir"}, "_", {"subst":"rpt"}, "/Aligned.toTranscriptome.out.bam" ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"chimeric_sam", + "required":"no", + "default":"star_out/Chimeric.out.sam", + "subst_constructor":{ + "vals":[ {"subst":"star_dir"}, "_", {"subst":"rpt"}, "/Chimeric.out.sam" ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"readspergene_tab", + "required":"no", + "default":"star_out/ReadsPerGene.out.tab", + "subst_constructor":{ + "vals":[ {"subst":"star_dir"}, "_", {"subst":"rpt"}, "/ReadsPerGene.out.tab" ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"cp_chimeric_sam_target", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".chimeric.sam" ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"cp_totranscriptome_bam_target", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".totranscriptome.bam" ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"cp_junctions_tab_target", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".junctions.tab" ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"cp_readspergene_tab_target", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".readspergene.tab" ], + "postproc":{"op":"concat","pad":""} + } + } +], +"nodes":[ + { + "id":"bamtofastq", + "type":"EXEC", + "use_STDIN": true, + "use_STDOUT": false, + "cmd":["bamtofastq", "gz=0", "F=__FQ1_OUT__", "F2=__FQ2_OUT__"] + }, + { + "id":"fq1", + "type":"RAFILE", + "name":{"subst":"fastq1"} + }, + { + "id":"fq2", + "type":"RAFILE", + "name":{"subst":"fastq2"} + }, + { + "id":"star", + "type":"EXEC", + "use_STDIN": false, + "use_STDOUT": true, + "cmd": [ + "STAR", + "--runMode", "alignReads", + "--outFileNamePrefix", {"subst":"star_out"}, + {"subst":"aligner_numthreads_flag"}, + "--genomeLoad", "NoSharedMemory", + {"subst":"sjdb_annotation_flag"} , + {"subst":"sjdb_overhang_flag"} , + "--outSAMstrandField", "intronMotif", + "--outSAMattributes", "NH", "HI", "NM", "MD", "AS", "XS", + "--outSAMunmapped", "Within", "KeepPairs", + "--outSAMtype", "BAM", "Unsorted", + "--outFilterIntronMotifs", "RemoveNoncanonicalUnannotated", + "--chimOutType", "SeparateSAMold", + "--chimSegmentMin", "15", + "--chimJunctionOverhangMin", "15", + "--quantMode", "TranscriptomeSAM", "GeneCounts", + "--genomeDir", "__REFERENCE_GENOME_IN__", + "--readFilesIn", "__FQ1_IN__", "__FQ2_IN__", + "--outStd", "BAM_Unsorted" + ] + }, + { + "id":"junctions_tab", + "type":"RAFILE", + "subtype":"DUMMY", + "name":{"subst":"junctions_tab"} + }, + { + "id":"totranscriptome_bam", + "type":"RAFILE", + "subtype":"DUMMY", + "name":{"subst":"totranscriptome_bam"} + }, + { + "id":"chimeric_sam", + "type":"RAFILE", + "subtype":"DUMMY", + "name":{"subst":"chimeric_sam"} + }, + { + "id":"readspergene_tab", + "type":"RAFILE", + "subtype":"DUMMY", + "name":{"subst":"readspergene_tab"} + }, + { + "id":"cp_chimeric_sam", + "type":"EXEC", + "use_STDIN": false, + "use_STDOUT": false, + "cmd":[ "cp", "__SRC_CHIMERIC_SAM_IN__", {"subst":"cp_chimeric_sam_target"} ] + }, + { + "id":"cp_totranscriptome_bam", + "type":"EXEC", + "use_STDIN": false, + "use_STDOUT": false, + "cmd":[ "cp", "__SRC_TOTRANSCRIPTOME_BAM_IN__", {"subst":"cp_totranscriptome_bam_target"} ] + }, + { + "id":"cp_junctions_tab", + "type":"EXEC", + "use_STDIN": false, + "use_STDOUT": false, + "cmd":[ "cp", "__SRC_JUNCTIONS_TAB_IN__", {"subst":"cp_junctions_tab_target"} ] + }, + { + "id":"cp_readspergene_tab", + "type":"EXEC", + "use_STDIN": false, + "use_STDOUT": false, + "cmd":[ "cp", "__SRC_READSPERGENE_TAB_IN__", {"subst":"cp_readspergene_tab_target"} ] + } +], +"edges":[ + { "id":"bamtofastq_to_fq1", "from":"bamtofastq:__FQ1_OUT__", "to":"fq1" }, + { "id":"bamtofastq_to_fq2", "from":"bamtofastq:__FQ2_OUT__", "to":"fq2" }, + { "id":"fq1_to_star", "from":"fq1", "to":"star:__FQ1_IN__" }, + { "id":"fq2_to_star", "from":"fq2", "to":"star:__FQ2_IN__" }, + { "id":"star_to_readspergene_tab", "from":"star", "to":"readspergene_tab" }, + { "id":"star_to_chimeric_sam", "from":"star", "to":"chimeric_sam" }, + { "id":"cp_chimeric_sam", "from":"chimeric_sam", "to":"cp_chimeric_sam:__SRC_CHIMERIC_SAM_IN__" }, + { "id":"star_to_totranscriptome_bam", "from":"star", "to":"totranscriptome_bam" }, + { "id":"cp_totranscriptome_bam", "from":"totranscriptome_bam", "to":"cp_totranscriptome_bam:__SRC_TOTRANSCRIPTOME_BAM_IN__" }, + { "id":"star_to_junctions_tab", "from":"star", "to":"junctions_tab" }, + { "id":"cp_junctions_tab", "from":"junctions_tab", "to":"cp_junctions_tab:__SRC_JUNCTIONS_TAB_IN__" }, + { "id":"star_to_readspergene_tab", "from":"star", "to":"readspergene_tab" }, + { "id":"cp_readspergene_tab", "from":"readspergene_tab", "to":"cp_readspergene_tab:__SRC_READSPERGENE_TAB_IN__" } +] +} \ No newline at end of file From 5fcc4853683139395dff33a22cadad1e12935183 Mon Sep 17 00:00:00 2001 From: "Ruben E. Bautista" Date: Fri, 10 Mar 2017 14:16:33 +0000 Subject: [PATCH 04/20] Convert sam- and bam-formatted outputs to cram - Convert these 2 output files: Chimeric.out.sam and Aligned.toTranscriptome.out.bam to CRAM using scramble - Note that for the transcriptome alignment a reference transcriptome fasta file is required --- data/vtlib/star_alignment.json | 50 +++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/data/vtlib/star_alignment.json b/data/vtlib/star_alignment.json index 945d16bf9..1d4ee4723 100644 --- a/data/vtlib/star_alignment.json +++ b/data/vtlib/star_alignment.json @@ -142,21 +142,31 @@ } }, { - "id":"cp_chimeric_sam_target", + "id":"scramble_chimeric_sam_target", "required":"no", "subst_constructor":{ - "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".chimeric.sam" ], + "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".chimeric.cram" ], "postproc":{"op":"concat","pad":""} } }, { - "id":"cp_totranscriptome_bam_target", + "id":"scramble_totranscriptome_bam_target", "required":"no", "subst_constructor":{ - "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".totranscriptome.bam" ], + "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".totranscriptome.cram" ], "postproc":{"op":"concat","pad":""} } }, + { + "id":"scramble_genome_reference_flag", + "required":"no", + "subst_constructor":{ "vals":[ "-r", {"subst":"reference_genome_fasta"} ] } + }, + { + "id":"scramble_transcriptome_reference_flag", + "required":"no", + "subst_constructor":{ "vals":[ "-r", {"subst":"reference_transcriptome_fasta"} ] } + }, { "id":"cp_junctions_tab_target", "required":"no", @@ -244,18 +254,38 @@ "name":{"subst":"readspergene_tab"} }, { - "id":"cp_chimeric_sam", + "id":"scramble_chimeric_sam", "type":"EXEC", "use_STDIN": false, "use_STDOUT": false, - "cmd":[ "cp", "__SRC_CHIMERIC_SAM_IN__", {"subst":"cp_chimeric_sam_target"} ] + "cmd":[ + "scramble", + {"subst":"b2c_mt", "ifnull":{"subst_constructor":{ "vals":[ "-t", {"subst":"b2c_mt_val"} ]}}}, + {"subst":"b2c_fmtver", "ifnull":{"subst_constructor":{ "vals":[ "-V", {"subst":"b2c_format_version"} ]}}}, + {"subst":"b2c_compress_level", "ifnull":"-7"}, + "-I", "sam", + "-O", "cram", + {"subst":"scramble_genome_reference_flag"}, + "__SRC_CHIMERIC_SAM_IN__", + {"subst":"scramble_chimeric_sam_target"} + ] }, { - "id":"cp_totranscriptome_bam", + "id":"scramble_totranscriptome_bam", "type":"EXEC", "use_STDIN": false, "use_STDOUT": false, - "cmd":[ "cp", "__SRC_TOTRANSCRIPTOME_BAM_IN__", {"subst":"cp_totranscriptome_bam_target"} ] + "cmd":[ + "scramble", + {"subst":"b2c_mt", "ifnull":{"subst_constructor":{ "vals":[ "-t", {"subst":"b2c_mt_val"} ]}}}, + {"subst":"b2c_fmtver", "ifnull":{"subst_constructor":{ "vals":[ "-V", {"subst":"b2c_format_version"} ]}}}, + {"subst":"b2c_compress_level", "ifnull":"-7"}, + "-I", "bam", + "-O", "cram", + {"subst":"scramble_transcriptome_reference_flag"}, + "__SRC_TOTRANSCRIPTOME_BAM_IN__", + {"subst":"scramble_totranscriptome_bam_target"} + ] }, { "id":"cp_junctions_tab", @@ -279,9 +309,9 @@ { "id":"fq2_to_star", "from":"fq2", "to":"star:__FQ2_IN__" }, { "id":"star_to_readspergene_tab", "from":"star", "to":"readspergene_tab" }, { "id":"star_to_chimeric_sam", "from":"star", "to":"chimeric_sam" }, - { "id":"cp_chimeric_sam", "from":"chimeric_sam", "to":"cp_chimeric_sam:__SRC_CHIMERIC_SAM_IN__" }, + { "id":"scramble_chimeric_sam", "from":"chimeric_sam", "to":"scramble_chimeric_sam:__SRC_CHIMERIC_SAM_IN__"}, { "id":"star_to_totranscriptome_bam", "from":"star", "to":"totranscriptome_bam" }, - { "id":"cp_totranscriptome_bam", "from":"totranscriptome_bam", "to":"cp_totranscriptome_bam:__SRC_TOTRANSCRIPTOME_BAM_IN__" }, + { "id":"scramble_totranscriptome_bam", "from":"totranscriptome_bam", "to":"scramble_totranscriptome_bam:__SRC_TOTRANSCRIPTOME_BAM_IN__" }, { "id":"star_to_junctions_tab", "from":"star", "to":"junctions_tab" }, { "id":"cp_junctions_tab", "from":"junctions_tab", "to":"cp_junctions_tab:__SRC_JUNCTIONS_TAB_IN__" }, { "id":"star_to_readspergene_tab", "from":"star", "to":"readspergene_tab" }, From f5a2c233efbd74aad812e60ebf37df192cb81a02 Mon Sep 17 00:00:00 2001 From: "Ruben E. Bautista" Date: Thu, 27 Jul 2017 13:53:07 +0100 Subject: [PATCH 05/20] Make ./ the default directory for star_dir; other improvements - Streamline the output files names defined by the argument --outFileNamePrefix to avoid name clashing when saving files in the current directory - Add a new star_executable parameter to specify a star binary other than the original 'STAR' e.g. 'star' --- data/vtlib/star_alignment.json | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/data/vtlib/star_alignment.json b/data/vtlib/star_alignment.json index 1d4ee4723..376b0016c 100644 --- a/data/vtlib/star_alignment.json +++ b/data/vtlib/star_alignment.json @@ -55,19 +55,19 @@ } }, { - "id":"star_dir","required":"no","default":"star_out" + "id":"star_dir","required":"no","default":"." }, { "id":"star_out", "required":"no", - "default": "star_out", "subst_constructor":{ - "vals":[ {"subst":"star_dir"}, "_", {"subst":"rpt"}, "/" ], + "vals":[ {"subst":"star_dir"}, "/", "_", {"subst":"rpt"}, "_" ], "postproc":{"op":"concat","pad":""} } }, { - "id":"transcriptome_subpath","required":"no" + "id":"transcriptome_subpath", + "required":"no" }, { "id":"sjdb_annotation_val", @@ -108,36 +108,32 @@ { "id":"junctions_tab", "required":"no", - "default":"star_out/SJ.out.tab", "subst_constructor":{ - "vals":[ {"subst":"star_dir"}, "_", {"subst":"rpt"}, "/SJ.out.tab" ], + "vals":[ {"subst":"star_out"}, "SJ.out.tab" ], "postproc":{"op":"concat","pad":""} } }, { "id":"totranscriptome_bam", "required":"no", - "default":"star_out/Aligned.toTranscriptome.out.bam", "subst_constructor":{ - "vals":[ {"subst":"star_dir"}, "_", {"subst":"rpt"}, "/Aligned.toTranscriptome.out.bam" ], + "vals":[ {"subst":"star_out"}, "Aligned.toTranscriptome.out.bam" ], "postproc":{"op":"concat","pad":""} } }, { "id":"chimeric_sam", "required":"no", - "default":"star_out/Chimeric.out.sam", "subst_constructor":{ - "vals":[ {"subst":"star_dir"}, "_", {"subst":"rpt"}, "/Chimeric.out.sam" ], + "vals":[ {"subst":"star_out"}, "Chimeric.out.sam" ], "postproc":{"op":"concat","pad":""} } }, { "id":"readspergene_tab", "required":"no", - "default":"star_out/ReadsPerGene.out.tab", "subst_constructor":{ - "vals":[ {"subst":"star_dir"}, "_", {"subst":"rpt"}, "/ReadsPerGene.out.tab" ], + "vals":[ {"subst":"star_out"}, "ReadsPerGene.out.tab" ], "postproc":{"op":"concat","pad":""} } }, @@ -182,6 +178,11 @@ "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".readspergene.tab" ], "postproc":{"op":"concat","pad":""} } + }, + { + "id":"star_executable", + "required":"no", + "default":"STAR" } ], "nodes":[ @@ -208,13 +209,13 @@ "use_STDIN": false, "use_STDOUT": true, "cmd": [ - "STAR", + {"subst":"star_executable"}, "--runMode", "alignReads", "--outFileNamePrefix", {"subst":"star_out"}, {"subst":"aligner_numthreads_flag"}, "--genomeLoad", "NoSharedMemory", - {"subst":"sjdb_annotation_flag"} , - {"subst":"sjdb_overhang_flag"} , + {"subst":"sjdb_annotation_flag"}, + {"subst":"sjdb_overhang_flag"}, "--outSAMstrandField", "intronMotif", "--outSAMattributes", "NH", "HI", "NM", "MD", "AS", "XS", "--outSAMunmapped", "Within", "KeepPairs", From 87d34463881083eb2c48196ee08aee070089f4b1 Mon Sep 17 00:00:00 2001 From: "Ruben E. Bautista" Date: Wed, 9 Aug 2017 10:33:27 +0100 Subject: [PATCH 06/20] Add Salmon quant using fastq files and transcriptome index as input - use salmon to quantify trancripts, input is fastq files aka quasi-mapping-mode (options -i, -1, -2) - zip a selection of Salmon's output files into a file in the output directory - add subt_params definitions for selected Salmon files - change the node sjdb_annotation_val to annotation_val to be reutilised by both star and salmon commands - include --geneMap option to Salmon command to produce quant files to the genome as well as to the transcriptome --- data/vtlib/star_alignment.json | 104 ++++++++++++++++++++++++++++++--- 1 file changed, 97 insertions(+), 7 deletions(-) diff --git a/data/vtlib/star_alignment.json b/data/vtlib/star_alignment.json index 376b0016c..5ae8a9327 100644 --- a/data/vtlib/star_alignment.json +++ b/data/vtlib/star_alignment.json @@ -66,11 +66,7 @@ } }, { - "id":"transcriptome_subpath", - "required":"no" - }, - { - "id":"sjdb_annotation_val", + "id":"annotation_val", "subst_constructor":{ "vals":[ {"subst":"reposdir"}, "/transcriptomes/", {"subst":"transcriptome_subpath"} ], "postproc":{"op":"concat","pad":""} @@ -80,7 +76,7 @@ "id":"sjdb_annotation_flag", "required":"no", "subst_constructor":{ - "vals":[ "--sjdbGTFfile", {"subst":"sjdb_annotation_val"} ], + "vals":[ "--sjdbGTFfile", {"subst":"annotation_val"} ], "postproc":{"op":"concat","pad":" "} } }, @@ -183,6 +179,64 @@ "id":"star_executable", "required":"no", "default":"STAR" + }, + { + "id":"salmon_dir", + "required":"no", + "default":"salmon_quant" + }, + { + "id":"salmon_out", + "required":"no", + "subst_constructor":{ "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"} ], "postproc":{"op":"concat","pad":""} } + }, + { + "id":"salmon_quant", + "required":"no", + "subst_constructor":{ "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/quant.sf" ], "postproc":{"op":"concat","pad":""} }, + "default":"salmon_quant/quant.sf" + }, + { + "id":"salmon_quant_genes", + "required":"no", + "subst_constructor":{ "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/quant.genes.sf" ], "postproc":{"op":"concat","pad":""} }, + "default":"salmon_quant/quant.genes.sf" + }, + { + "id":"salmon_lib_format_counts", + "required":"no", + "subst_constructor":{ "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/lib_format_counts.json" ], "postproc":{"op":"concat","pad":""} }, + "default":"salmon_quant/lib_format_counts.json" + }, + { + "id":"salmon_libparams", + "required":"no", + "subst_constructor":{ "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/libParams" ], "postproc":{"op":"concat","pad":""} }, + "default":"salmon_quant/libParams" + }, + { + "id":"salmon_cmd_info", + "required":"no", + "subst_constructor":{ "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/cmd_info.json" ], "postproc":{"op":"concat","pad":""} }, + "default":"salmon_quant/cmd_info.json" + }, + { + "id":"zip_salmon_quant_target", + "required":"no", + "subst_constructor":{ "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".salmon_quant.zip" ], "postproc":{"op":"concat","pad":""} } + }, + { + "id":"transcriptome_subpath", + "required":"no" + }, + { + "id":"transcriptome_val", + "subst_constructor":{ "vals":[ {"subst":"reposdir"}, "/transcriptomes/", {"subst":"transcriptome_subpath"} ], "postproc":{"op":"concat","pad":""} } + }, + { + "id":"gene_mapping_flag", + "required":"no", + "subst_constructor":{ "vals":[ "--geneMap=", {"subst":"annotation_val"} ], "postproc":{"op":"concat","pad":""} } } ], "nodes":[ @@ -301,6 +355,39 @@ "use_STDIN": false, "use_STDOUT": false, "cmd":[ "cp", "__SRC_READSPERGENE_TAB_IN__", {"subst":"cp_readspergene_tab_target"} ] + }, + { + "id":"salmon", + "type":"EXEC", + "use_STDIN": false, + "use_STDOUT": true, + "cmd":[ + "salmon", + "--no-version-check", + "quant", + "--index", {"subst":"transcriptome_val"}, + "--libType", "A", + "--mates1", "__FQ1_IN__", + "--mates2", "__FQ2_IN__", + {"subst":"gene_mapping_flag"}, + {"subst":"b2c_mt", "ifnull":{"subst_constructor":{ "vals":[ "-p", {"subst":"b2c_mt_val"} ]}}}, + "--output", {"subst":"salmon_out"} + ] + }, + { + "id":"zip_salmon_quant", + "type":"EXEC", + "use_STDIN": true, + "use_STDOUT": false, + "cmd":[ + "zip", "-r", + {"subst":"zip_salmon_quant_target"}, + {"subst":"salmon_quant"}, + {"subst":"salmon_quant_genes"}, + {"subst":"salmon_lib_format_counts"}, + {"subst":"salmon_libparams"}, + {"subst":"salmon_cmd_info"} + ] } ], "edges":[ @@ -316,6 +403,9 @@ { "id":"star_to_junctions_tab", "from":"star", "to":"junctions_tab" }, { "id":"cp_junctions_tab", "from":"junctions_tab", "to":"cp_junctions_tab:__SRC_JUNCTIONS_TAB_IN__" }, { "id":"star_to_readspergene_tab", "from":"star", "to":"readspergene_tab" }, - { "id":"cp_readspergene_tab", "from":"readspergene_tab", "to":"cp_readspergene_tab:__SRC_READSPERGENE_TAB_IN__" } + { "id":"cp_readspergene_tab", "from":"readspergene_tab", "to":"cp_readspergene_tab:__SRC_READSPERGENE_TAB_IN__" }, + { "id":"fq1_to_salmon", "from":"fq1", "to":"salmon:__FQ1_IN__" }, + { "id":"fq2_to_salmon", "from":"fq2", "to":"salmon:__FQ2_IN__" }, + { "id":"salmon_to_zip_salmon_quant", "from":"salmon", "to":"zip_salmon_quant"} ] } \ No newline at end of file From c670f90f4b17bb807c8e0750226ae4ffef9f97ac Mon Sep 17 00:00:00 2001 From: "Ruben E. Bautista" Date: Wed, 9 Aug 2017 17:05:41 +0100 Subject: [PATCH 07/20] Add Salmon quant using fastq files as input - other inputs: annotation in gtf format and transcriptome index generated using Salmon indexing command --- data/vtlib/tophat2_alignment.json | 128 ++++++++++++++++++++++++++++-- 1 file changed, 121 insertions(+), 7 deletions(-) diff --git a/data/vtlib/tophat2_alignment.json b/data/vtlib/tophat2_alignment.json index 7add77269..a18fa0eaa 100644 --- a/data/vtlib/tophat2_alignment.json +++ b/data/vtlib/tophat2_alignment.json @@ -13,13 +13,13 @@ } }, "subst_params":[ - { - "id": "basic_pipeline_params", - "type":"SPFILE", + { + "id": "basic_pipeline_params", + "type":"SPFILE", "name":{"subst":"basic_pipeline_params_file"}, - "required": "no", - "comment":"this will expand to a set of subst_param elements" - }, + "required": "no", + "comment":"this will expand to a set of subst_param elements" + }, { "id":"fastq1_name", "required":"no", @@ -166,6 +166,84 @@ "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".junctions.bed" ], "postproc":{"op":"concat","pad":""} } + }, + { + "id":"salmon_dir", + "required":"no", + "default":"salmon_quant" + }, + { + "id":"salmon_out", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"} ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"salmon_quant", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/quant.sf" ], + "postproc":{"op":"concat","pad":""} + }, + "default":"salmon_quant/quant.sf" + }, + { + "id":"salmon_quant_genes", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/quant.genes.sf" ], + "postproc":{"op":"concat","pad":""} + }, + "default":"salmon_quant/quant.genes.sf" + }, + { + "id":"salmon_lib_format_counts", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/lib_format_counts.json" ], + "postproc":{"op":"concat","pad":""} + }, + "default":"salmon_quant/lib_format_counts.json" + }, + { + "id":"salmon_libparams", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/libParams" ], + "postproc":{"op":"concat","pad":""} + }, + "default":"salmon_quant/libParams" + }, + { + "id":"salmon_cmd_info", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/cmd_info.json" ], + "postproc":{"op":"concat","pad":""} + }, + "default":"salmon_quant/cmd_info.json" + }, + { + "id":"zip_salmon_quant_target", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".salmon_quant.zip" ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"gene_mapping_flag", + "required":"no", + "subst_constructor":{ + "vals":[ "--geneMap=", {"subst":"annotation_val"} ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"salmon_transcriptome_val", + "required":"yes" } ], "nodes":[ @@ -265,6 +343,39 @@ "use_STDIN": false, "use_STDOUT": false, "cmd":[ "cp", "__SRC_JUNCTIONS_BED_IN__", {"subst":"cp_junctions_bed_target"} ] + }, + { + "id":"salmon", + "type":"EXEC", + "use_STDIN": false, + "use_STDOUT": true, + "cmd":[ + "salmon", + "--no-version-check", + "quant", + "--index", {"subst":"salmon_transcriptome_val"}, + "--libType", "A", + "--mates1", "__FQ1_IN__", + "--mates2", "__FQ2_IN__", + {"subst":"gene_mapping_flag"}, + {"subst":"b2c_mt", "ifnull":{"subst_constructor":{ "vals":[ "-p", {"subst":"b2c_mt_val"} ]}}}, + "--output", {"subst":"salmon_out"} + ] + }, + { + "id":"zip_salmon_quant", + "type":"EXEC", + "use_STDIN": true, + "use_STDOUT": false, + "cmd":[ + "zip", "-r", + {"subst":"zip_salmon_quant_target"}, + {"subst":"salmon_quant"}, + {"subst":"salmon_quant_genes"}, + {"subst":"salmon_lib_format_counts"}, + {"subst":"salmon_libparams"}, + {"subst":"salmon_cmd_info"} + ] } ], "edges":[ @@ -281,6 +392,9 @@ { "id":"tophat2_to_junctions_bed", "from":"tophat2", "to":"junctions_bed" }, { "id":"cp_junctions_bed", "from":"junctions_bed", "to":"cp_junctions_bed:__SRC_JUNCTIONS_BED_IN__" }, { "id":"accepted_hits_bam_to_bamcat", "from":"accepted_hits_bam", "to":"bamcat:__IN_BAM1__" }, - { "id":"unmapped_bam_to_bamcat", "from":"unmapped_bam", "to":"bamcat:__IN_BAM2__" } + { "id":"unmapped_bam_to_bamcat", "from":"unmapped_bam", "to":"bamcat:__IN_BAM2__" }, + { "id":"fq1_to_salmon", "from":"fq1", "to":"salmon:__FQ1_IN__" }, + { "id":"fq2_to_salmon", "from":"fq2", "to":"salmon:__FQ2_IN__" }, + { "id":"salmon_to_zip_salmon_quant", "from":"salmon", "to":"zip_salmon_quant"} ] } From 5b70d6f2830f16948b76bc06807273e42250bb9c Mon Sep 17 00:00:00 2001 From: "Ruben E. Bautista" Date: Thu, 17 Aug 2017 14:05:55 +0100 Subject: [PATCH 08/20] Run Salmon from Star using a new P4 template - create a new template: salmon_alignment.json - remove salmon nodes and edges from star template - add a vtfile node to star template --- data/vtlib/salmon_alignment.json | 130 +++++++++++++++++++++++++++++++ data/vtlib/star_alignment.json | 102 ++++-------------------- 2 files changed, 144 insertions(+), 88 deletions(-) create mode 100644 data/vtlib/salmon_alignment.json diff --git a/data/vtlib/salmon_alignment.json b/data/vtlib/salmon_alignment.json new file mode 100644 index 000000000..1bb22432f --- /dev/null +++ b/data/vtlib/salmon_alignment.json @@ -0,0 +1,130 @@ +{ +"version":"1.0", +"description":"steps in the alignment pipeline perform a checksum-based comparison of input and output (bam) data. Final validation step in alignment pipeline", +"subgraph_io":{ + "ports":{ + "inputs":{ + "fq1":"salmon:__FQ1_IN__", + "fq2":"salmon:__FQ2_IN__" + } + } +}, +"subst_params":[ + { + "id":"salmon_dir", + "required":"no", + "default":"salmon_quant" + }, + { + "id":"salmon_out", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"} ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"quant", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/quant.sf" ], + "postproc":{"op":"concat","pad":""} + }, + "default":"salmon_quant/quant.sf" + }, + { + "id":"quant_genes", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/quant.genes.sf" ], + "postproc":{"op":"concat","pad":""} + }, + "default":"salmon_quant/quant.genes.sf" + }, + { + "id":"lib_format_counts", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/lib_format_counts.json" ], + "postproc":{"op":"concat","pad":""} + }, + "default":"salmon_quant/lib_format_counts.json" + }, + { + "id":"libparams", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/libParams" ], + "postproc":{"op":"concat","pad":""} + }, + "default":"salmon_quant/libParams" + }, + { + "id":"cmd_info", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/cmd_info.json" ], + "postproc":{"op":"concat","pad":""} + }, + "default":"salmon_quant/cmd_info.json" + }, + { + "id":"zip_target", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".salmon_quant.zip" ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"gene_mapping_flag", + "required":"no", + "subst_constructor":{ + "vals":[ "--geneMap=", {"subst":"annotation_val"} ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"salmon_transcriptome_val", + "required":"yes" + } +], +"nodes":[ + { + "id":"salmon", + "type":"EXEC", + "use_STDIN": false, + "use_STDOUT": true, + "cmd":[ + "salmon", + "--no-version-check", + "quant", + "--index", {"subst":"salmon_transcriptome_val"}, + "--libType", "A", + "--mates1", "__FQ1_IN__", + "--mates2", "__FQ2_IN__", + {"subst":"gene_mapping_flag"}, + {"subst":"b2c_mt", "ifnull":{"subst_constructor":{ "vals":[ "-p", {"subst":"b2c_mt_val"} ]}}}, + "--output", {"subst":"salmon_out"} + ] + }, + { + "id":"zip_salmon_quant", + "type":"EXEC", + "use_STDIN": true, + "use_STDOUT": false, + "cmd":[ + "zip", "-r", + {"subst":"zip_target"}, + {"subst":"quant"}, + {"subst":"quant_genes"}, + {"subst":"lib_format_counts"}, + {"subst":"libparams"}, + {"subst":"cmd_info"} + ] + } +], +"edges":[ + { "id":"salmon_to_zip_salmon_quant", "from":"salmon", "to":"zip_salmon_quant"} +] +} \ No newline at end of file diff --git a/data/vtlib/star_alignment.json b/data/vtlib/star_alignment.json index 5ae8a9327..1ad9f76d6 100644 --- a/data/vtlib/star_alignment.json +++ b/data/vtlib/star_alignment.json @@ -181,62 +181,12 @@ "default":"STAR" }, { - "id":"salmon_dir", - "required":"no", - "default":"salmon_quant" - }, - { - "id":"salmon_out", - "required":"no", - "subst_constructor":{ "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"} ], "postproc":{"op":"concat","pad":""} } - }, - { - "id":"salmon_quant", - "required":"no", - "subst_constructor":{ "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/quant.sf" ], "postproc":{"op":"concat","pad":""} }, - "default":"salmon_quant/quant.sf" - }, - { - "id":"salmon_quant_genes", - "required":"no", - "subst_constructor":{ "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/quant.genes.sf" ], "postproc":{"op":"concat","pad":""} }, - "default":"salmon_quant/quant.genes.sf" - }, - { - "id":"salmon_lib_format_counts", - "required":"no", - "subst_constructor":{ "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/lib_format_counts.json" ], "postproc":{"op":"concat","pad":""} }, - "default":"salmon_quant/lib_format_counts.json" - }, - { - "id":"salmon_libparams", - "required":"no", - "subst_constructor":{ "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/libParams" ], "postproc":{"op":"concat","pad":""} }, - "default":"salmon_quant/libParams" - }, - { - "id":"salmon_cmd_info", - "required":"no", - "subst_constructor":{ "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/cmd_info.json" ], "postproc":{"op":"concat","pad":""} }, - "default":"salmon_quant/cmd_info.json" - }, - { - "id":"zip_salmon_quant_target", - "required":"no", - "subst_constructor":{ "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".salmon_quant.zip" ], "postproc":{"op":"concat","pad":""} } - }, - { - "id":"transcriptome_subpath", - "required":"no" - }, - { - "id":"transcriptome_val", - "subst_constructor":{ "vals":[ {"subst":"reposdir"}, "/transcriptomes/", {"subst":"transcriptome_subpath"} ], "postproc":{"op":"concat","pad":""} } - }, - { - "id":"gene_mapping_flag", - "required":"no", - "subst_constructor":{ "vals":[ "--geneMap=", {"subst":"annotation_val"} ], "postproc":{"op":"concat","pad":""} } + "id":"quant_vtf", + "required":"yes", + "subst_constructor":{ + "vals":[ {"subst":"cfgdatadir"}, "/", {"subst":"quant_method"}, "_alignment.json" ], + "postproc":{"op":"concat", "pad":""} + } } ], "nodes":[ @@ -357,37 +307,14 @@ "cmd":[ "cp", "__SRC_READSPERGENE_TAB_IN__", {"subst":"cp_readspergene_tab_target"} ] }, { - "id":"salmon", - "type":"EXEC", + "id":"quantify", + "type":"VTFILE", "use_STDIN": false, "use_STDOUT": true, - "cmd":[ - "salmon", - "--no-version-check", - "quant", - "--index", {"subst":"transcriptome_val"}, - "--libType", "A", - "--mates1", "__FQ1_IN__", - "--mates2", "__FQ2_IN__", - {"subst":"gene_mapping_flag"}, - {"subst":"b2c_mt", "ifnull":{"subst_constructor":{ "vals":[ "-p", {"subst":"b2c_mt_val"} ]}}}, - "--output", {"subst":"salmon_out"} - ] - }, - { - "id":"zip_salmon_quant", - "type":"EXEC", - "use_STDIN": true, - "use_STDOUT": false, - "cmd":[ - "zip", "-r", - {"subst":"zip_salmon_quant_target"}, - {"subst":"salmon_quant"}, - {"subst":"salmon_quant_genes"}, - {"subst":"salmon_lib_format_counts"}, - {"subst":"salmon_libparams"}, - {"subst":"salmon_cmd_info"} - ] + "comment":"inputs: fq1, fq2; outputs: NONE", + "node_prefix":"quant_", + "name":{"subst":"quant_vtf"}, + "description":"subgraph containing salmon quantification of transcripts" } ], "edges":[ @@ -404,8 +331,7 @@ { "id":"cp_junctions_tab", "from":"junctions_tab", "to":"cp_junctions_tab:__SRC_JUNCTIONS_TAB_IN__" }, { "id":"star_to_readspergene_tab", "from":"star", "to":"readspergene_tab" }, { "id":"cp_readspergene_tab", "from":"readspergene_tab", "to":"cp_readspergene_tab:__SRC_READSPERGENE_TAB_IN__" }, - { "id":"fq1_to_salmon", "from":"fq1", "to":"salmon:__FQ1_IN__" }, - { "id":"fq2_to_salmon", "from":"fq2", "to":"salmon:__FQ2_IN__" }, - { "id":"salmon_to_zip_salmon_quant", "from":"salmon", "to":"zip_salmon_quant"} + { "id":"fq1_to_quantify", "from":"fq1", "to":"quantify:fq1" }, + { "id":"fq2_to_quantify", "from":"fq2", "to":"quantify:fq2" } ] } \ No newline at end of file From 8d189c9587fe5345e1c269c1623ce6e8106372a9 Mon Sep 17 00:00:00 2001 From: "Ruben E. Bautista" Date: Mon, 21 Aug 2017 11:18:29 +0100 Subject: [PATCH 09/20] Change name of input ports in salmon template to improve clarity - other fixes and improvements --- data/vtlib/salmon_alignment.json | 4 ++-- data/vtlib/star_alignment.json | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/data/vtlib/salmon_alignment.json b/data/vtlib/salmon_alignment.json index 1bb22432f..9a081c972 100644 --- a/data/vtlib/salmon_alignment.json +++ b/data/vtlib/salmon_alignment.json @@ -4,8 +4,8 @@ "subgraph_io":{ "ports":{ "inputs":{ - "fq1":"salmon:__FQ1_IN__", - "fq2":"salmon:__FQ2_IN__" + "fastq1":"salmon:__FQ1_IN__", + "fastq2":"salmon:__FQ2_IN__" } } }, diff --git a/data/vtlib/star_alignment.json b/data/vtlib/star_alignment.json index 1ad9f76d6..4469a5c94 100644 --- a/data/vtlib/star_alignment.json +++ b/data/vtlib/star_alignment.json @@ -150,9 +150,9 @@ } }, { - "id":"scramble_genome_reference_flag", + "id":"scramble_chimeric_reference_flag", "required":"no", - "subst_constructor":{ "vals":[ "-r", {"subst":"reference_genome_fasta"} ] } + "subst_constructor":{ "vals":[ "-r", {"subst":"scramble_reference_fasta"} ] } }, { "id":"scramble_transcriptome_reference_flag", @@ -270,7 +270,7 @@ {"subst":"b2c_compress_level", "ifnull":"-7"}, "-I", "sam", "-O", "cram", - {"subst":"scramble_genome_reference_flag"}, + {"subst":"scramble_chimeric_reference_flag"}, "__SRC_CHIMERIC_SAM_IN__", {"subst":"scramble_chimeric_sam_target"} ] @@ -322,7 +322,6 @@ { "id":"bamtofastq_to_fq2", "from":"bamtofastq:__FQ2_OUT__", "to":"fq2" }, { "id":"fq1_to_star", "from":"fq1", "to":"star:__FQ1_IN__" }, { "id":"fq2_to_star", "from":"fq2", "to":"star:__FQ2_IN__" }, - { "id":"star_to_readspergene_tab", "from":"star", "to":"readspergene_tab" }, { "id":"star_to_chimeric_sam", "from":"star", "to":"chimeric_sam" }, { "id":"scramble_chimeric_sam", "from":"chimeric_sam", "to":"scramble_chimeric_sam:__SRC_CHIMERIC_SAM_IN__"}, { "id":"star_to_totranscriptome_bam", "from":"star", "to":"totranscriptome_bam" }, @@ -331,7 +330,7 @@ { "id":"cp_junctions_tab", "from":"junctions_tab", "to":"cp_junctions_tab:__SRC_JUNCTIONS_TAB_IN__" }, { "id":"star_to_readspergene_tab", "from":"star", "to":"readspergene_tab" }, { "id":"cp_readspergene_tab", "from":"readspergene_tab", "to":"cp_readspergene_tab:__SRC_READSPERGENE_TAB_IN__" }, - { "id":"fq1_to_quantify", "from":"fq1", "to":"quantify:fq1" }, - { "id":"fq2_to_quantify", "from":"fq2", "to":"quantify:fq2" } + { "id":"fq1_to_quantify", "from":"fq1", "to":"quantify:fastq1" }, + { "id":"fq2_to_quantify", "from":"fq2", "to":"quantify:fastq2" } ] } \ No newline at end of file From c7c44d92f6a3936b4f45efb38297333696d12610 Mon Sep 17 00:00:00 2001 From: "Ruben E. Bautista" Date: Mon, 21 Aug 2017 11:49:52 +0100 Subject: [PATCH 10/20] Run Salmon quantification in Tophat2 using a template - remove salmon from tophat2 template and use a template that can be used by other templates e.g. star alignment - add new template file to run salmon quantification: salmon_alignment.json --- data/vtlib/salmon_alignment.json | 130 ++++++++++++++++++++++++++++++ data/vtlib/tophat2_alignment.json | 118 +++------------------------ 2 files changed, 142 insertions(+), 106 deletions(-) create mode 100644 data/vtlib/salmon_alignment.json diff --git a/data/vtlib/salmon_alignment.json b/data/vtlib/salmon_alignment.json new file mode 100644 index 000000000..9a081c972 --- /dev/null +++ b/data/vtlib/salmon_alignment.json @@ -0,0 +1,130 @@ +{ +"version":"1.0", +"description":"steps in the alignment pipeline perform a checksum-based comparison of input and output (bam) data. Final validation step in alignment pipeline", +"subgraph_io":{ + "ports":{ + "inputs":{ + "fastq1":"salmon:__FQ1_IN__", + "fastq2":"salmon:__FQ2_IN__" + } + } +}, +"subst_params":[ + { + "id":"salmon_dir", + "required":"no", + "default":"salmon_quant" + }, + { + "id":"salmon_out", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"} ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"quant", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/quant.sf" ], + "postproc":{"op":"concat","pad":""} + }, + "default":"salmon_quant/quant.sf" + }, + { + "id":"quant_genes", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/quant.genes.sf" ], + "postproc":{"op":"concat","pad":""} + }, + "default":"salmon_quant/quant.genes.sf" + }, + { + "id":"lib_format_counts", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/lib_format_counts.json" ], + "postproc":{"op":"concat","pad":""} + }, + "default":"salmon_quant/lib_format_counts.json" + }, + { + "id":"libparams", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/libParams" ], + "postproc":{"op":"concat","pad":""} + }, + "default":"salmon_quant/libParams" + }, + { + "id":"cmd_info", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/cmd_info.json" ], + "postproc":{"op":"concat","pad":""} + }, + "default":"salmon_quant/cmd_info.json" + }, + { + "id":"zip_target", + "required":"no", + "subst_constructor":{ + "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".salmon_quant.zip" ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"gene_mapping_flag", + "required":"no", + "subst_constructor":{ + "vals":[ "--geneMap=", {"subst":"annotation_val"} ], + "postproc":{"op":"concat","pad":""} + } + }, + { + "id":"salmon_transcriptome_val", + "required":"yes" + } +], +"nodes":[ + { + "id":"salmon", + "type":"EXEC", + "use_STDIN": false, + "use_STDOUT": true, + "cmd":[ + "salmon", + "--no-version-check", + "quant", + "--index", {"subst":"salmon_transcriptome_val"}, + "--libType", "A", + "--mates1", "__FQ1_IN__", + "--mates2", "__FQ2_IN__", + {"subst":"gene_mapping_flag"}, + {"subst":"b2c_mt", "ifnull":{"subst_constructor":{ "vals":[ "-p", {"subst":"b2c_mt_val"} ]}}}, + "--output", {"subst":"salmon_out"} + ] + }, + { + "id":"zip_salmon_quant", + "type":"EXEC", + "use_STDIN": true, + "use_STDOUT": false, + "cmd":[ + "zip", "-r", + {"subst":"zip_target"}, + {"subst":"quant"}, + {"subst":"quant_genes"}, + {"subst":"lib_format_counts"}, + {"subst":"libparams"}, + {"subst":"cmd_info"} + ] + } +], +"edges":[ + { "id":"salmon_to_zip_salmon_quant", "from":"salmon", "to":"zip_salmon_quant"} +] +} \ No newline at end of file diff --git a/data/vtlib/tophat2_alignment.json b/data/vtlib/tophat2_alignment.json index a18fa0eaa..2f2badf07 100644 --- a/data/vtlib/tophat2_alignment.json +++ b/data/vtlib/tophat2_alignment.json @@ -168,82 +168,12 @@ } }, { - "id":"salmon_dir", - "required":"no", - "default":"salmon_quant" - }, - { - "id":"salmon_out", - "required":"no", - "subst_constructor":{ - "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"} ], - "postproc":{"op":"concat","pad":""} - } - }, - { - "id":"salmon_quant", - "required":"no", - "subst_constructor":{ - "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/quant.sf" ], - "postproc":{"op":"concat","pad":""} - }, - "default":"salmon_quant/quant.sf" - }, - { - "id":"salmon_quant_genes", - "required":"no", - "subst_constructor":{ - "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/quant.genes.sf" ], - "postproc":{"op":"concat","pad":""} - }, - "default":"salmon_quant/quant.genes.sf" - }, - { - "id":"salmon_lib_format_counts", - "required":"no", - "subst_constructor":{ - "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/lib_format_counts.json" ], - "postproc":{"op":"concat","pad":""} - }, - "default":"salmon_quant/lib_format_counts.json" - }, - { - "id":"salmon_libparams", - "required":"no", - "subst_constructor":{ - "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/libParams" ], - "postproc":{"op":"concat","pad":""} - }, - "default":"salmon_quant/libParams" - }, - { - "id":"salmon_cmd_info", - "required":"no", - "subst_constructor":{ - "vals":[ {"subst":"salmon_dir"}, "_", {"subst":"rpt"}, "/cmd_info.json" ], - "postproc":{"op":"concat","pad":""} - }, - "default":"salmon_quant/cmd_info.json" - }, - { - "id":"zip_salmon_quant_target", - "required":"no", - "subst_constructor":{ - "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".salmon_quant.zip" ], - "postproc":{"op":"concat","pad":""} - } - }, - { - "id":"gene_mapping_flag", - "required":"no", + "id":"quant_vtf", + "required":"yes", "subst_constructor":{ - "vals":[ "--geneMap=", {"subst":"annotation_val"} ], - "postproc":{"op":"concat","pad":""} + "vals":[ {"subst":"cfgdatadir"}, "/", {"subst":"quant_method"}, "_alignment.json" ], + "postproc":{"op":"concat", "pad":""} } - }, - { - "id":"salmon_transcriptome_val", - "required":"yes" } ], "nodes":[ @@ -345,37 +275,14 @@ "cmd":[ "cp", "__SRC_JUNCTIONS_BED_IN__", {"subst":"cp_junctions_bed_target"} ] }, { - "id":"salmon", - "type":"EXEC", + "id":"quantify", + "type":"VTFILE", "use_STDIN": false, "use_STDOUT": true, - "cmd":[ - "salmon", - "--no-version-check", - "quant", - "--index", {"subst":"salmon_transcriptome_val"}, - "--libType", "A", - "--mates1", "__FQ1_IN__", - "--mates2", "__FQ2_IN__", - {"subst":"gene_mapping_flag"}, - {"subst":"b2c_mt", "ifnull":{"subst_constructor":{ "vals":[ "-p", {"subst":"b2c_mt_val"} ]}}}, - "--output", {"subst":"salmon_out"} - ] - }, - { - "id":"zip_salmon_quant", - "type":"EXEC", - "use_STDIN": true, - "use_STDOUT": false, - "cmd":[ - "zip", "-r", - {"subst":"zip_salmon_quant_target"}, - {"subst":"salmon_quant"}, - {"subst":"salmon_quant_genes"}, - {"subst":"salmon_lib_format_counts"}, - {"subst":"salmon_libparams"}, - {"subst":"salmon_cmd_info"} - ] + "comment":"inputs: fastq1, fastq2; outputs: NONE", + "node_prefix":"quant_", + "name":{"subst":"quant_vtf"}, + "description":"subgraph containing salmon quantification of transcripts" } ], "edges":[ @@ -393,8 +300,7 @@ { "id":"cp_junctions_bed", "from":"junctions_bed", "to":"cp_junctions_bed:__SRC_JUNCTIONS_BED_IN__" }, { "id":"accepted_hits_bam_to_bamcat", "from":"accepted_hits_bam", "to":"bamcat:__IN_BAM1__" }, { "id":"unmapped_bam_to_bamcat", "from":"unmapped_bam", "to":"bamcat:__IN_BAM2__" }, - { "id":"fq1_to_salmon", "from":"fq1", "to":"salmon:__FQ1_IN__" }, - { "id":"fq2_to_salmon", "from":"fq2", "to":"salmon:__FQ2_IN__" }, - { "id":"salmon_to_zip_salmon_quant", "from":"salmon", "to":"zip_salmon_quant"} + { "id":"fq1_to_quantify", "from":"fq1", "to":"quantify:fastq1" }, + { "id":"fq2_to_quantify", "from":"fq2", "to":"quantify:fastq2" } ] } From 0dd038de356c2a421ed02206f7f6d4b4a1d18d95 Mon Sep 17 00:00:00 2001 From: "Ruben E. Bautista" Date: Mon, 4 Sep 2017 11:27:54 +0100 Subject: [PATCH 11/20] Add option -G to tophat2 cmd to be used if transcriptome index is missing - If --transcriptome-index cannot be used because transcriptome_flag is undef (i.e. no "known" transcripts index file is used) then use the option --GTF instead by adding an "ifnull" to the transcriptome_flag key that will replace it with annotation_flag so even if there is no transcriptome index available an annotation can be passed to tophat2 to guide the alignment to the transcriptome. This will have the side-effect of making the execution of tophat2 slower as it will have to generate a temporary transcriptome index locally. This isn't ideal, but the lack of an index file doesn't prevent the alignment from running successfully. - This change is related to PR 235 of wtsi-npg/npg_seq_pipeline --- data/vtlib/tophat2_alignment.json | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/data/vtlib/tophat2_alignment.json b/data/vtlib/tophat2_alignment.json index 2f2badf07..cfa453149 100644 --- a/data/vtlib/tophat2_alignment.json +++ b/data/vtlib/tophat2_alignment.json @@ -80,6 +80,13 @@ "postproc":{"op":"concat","pad":"="} } }, + { + "id":"annotation_val", + "subst_constructor":{ + "vals":[ {"subst":"reposdir"}, "/transcriptomes/", {"subst":"transcriptome_subpath"} ], + "postproc":{"op":"concat","pad":""} + } + }, { "id":"aligner_numthreads_flag", "required":"no", @@ -96,6 +103,14 @@ "postproc":{"op":"concat","pad":"="} } }, + { + "id":"annotation_flag", + "required":"no", + "subst_constructor":{ + "vals":[ "--GTF", {"subst":"annotation_val"} ], + "postproc":{"op":"concat","pad":"="} + } + }, { "id":"junctions_bed", "required":"no", @@ -208,9 +223,9 @@ "--mate-inner-dist","100", {"subst":"aligner_numthreads_flag"}, {"subst":"library_type_flag"}, + {"subst":"transcriptome_flag", "ifnull":{"subst":"annotation_flag"}}, "--no-coverage-search", "--microexon-search", - {"subst":"transcriptome_flag"} , "__REFERENCE_GENOME_IN__", "__FQ1_IN__", "__FQ2_IN__" From 335327f9248833e69b58775a19d65701ad0e2993 Mon Sep 17 00:00:00 2001 From: "Ruben E. Bautista" Date: Mon, 4 Sep 2017 14:21:02 +0100 Subject: [PATCH 12/20] Add genome and transcriptome fasta keys that are required for this template - the option -r in the scramble commands that use the genome and transcriptome files are required so the use of a "flag" keys isn't appropriate as they are used for optional command arguments. --- data/vtlib/star_alignment.json | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/data/vtlib/star_alignment.json b/data/vtlib/star_alignment.json index 4469a5c94..90defe7ef 100644 --- a/data/vtlib/star_alignment.json +++ b/data/vtlib/star_alignment.json @@ -150,14 +150,24 @@ } }, { - "id":"scramble_chimeric_reference_flag", - "required":"no", - "subst_constructor":{ "vals":[ "-r", {"subst":"scramble_reference_fasta"} ] } + "id":"reference_genome_fasta_name","required":"no"}, + { + "id":"reference_genome_fasta", + "required":"yes", + "subst_constructor":{ + "vals":[ {"subst":"reposdir"}, "/", {"subst":"reference_genome_fasta_name"} ], + "postproc":{"op":"concat", "pad":""} + } }, { - "id":"scramble_transcriptome_reference_flag", - "required":"no", - "subst_constructor":{ "vals":[ "-r", {"subst":"reference_transcriptome_fasta"} ] } + "id":"reference_transcriptome_fasta_name","required":"no"}, + { + "id":"reference_transcriptome_fasta", + "required":"yes", + "subst_constructor":{ + "vals":[ {"subst":"reposdir"}, "/", {"subst":"reference_transcriptome_fasta_name"} ], + "postproc":{"op":"concat", "pad":""} + } }, { "id":"cp_junctions_tab_target", @@ -270,7 +280,7 @@ {"subst":"b2c_compress_level", "ifnull":"-7"}, "-I", "sam", "-O", "cram", - {"subst":"scramble_chimeric_reference_flag"}, + "-r", {"subst":"reference_genome_fasta"}, "__SRC_CHIMERIC_SAM_IN__", {"subst":"scramble_chimeric_sam_target"} ] @@ -287,7 +297,7 @@ {"subst":"b2c_compress_level", "ifnull":"-7"}, "-I", "bam", "-O", "cram", - {"subst":"scramble_transcriptome_reference_flag"}, + "-r", {"subst":"reference_transcriptome_fasta"}, "__SRC_TOTRANSCRIPTOME_BAM_IN__", {"subst":"scramble_totranscriptome_bam_target"} ] From 833bc5f2f2b84e7369f476befa5c0f06633580e6 Mon Sep 17 00:00:00 2001 From: Carol Scott Date: Thu, 21 Sep 2017 15:45:32 +0100 Subject: [PATCH 13/20] Small change in final_output_prep to allow bamsort_cmd to have configurable executable --- Changes | 2 ++ data/vtlib/final_output_prep.json | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Changes b/Changes index 36fc50ce2..4c81d362c 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,8 @@ CHANGES LOG ----------- + - Small change in final_output_prep to allow bamsort_cmd to have configurable executable. + release 0.18.6 - Y-split fixes When processing VTFILE nodes, make sure that parameters are reevaluated locally instead of naively inheriting values diff --git a/data/vtlib/final_output_prep.json b/data/vtlib/final_output_prep.json index 33e89e932..18765920d 100644 --- a/data/vtlib/final_output_prep.json +++ b/data/vtlib/final_output_prep.json @@ -238,7 +238,7 @@ "type":"EXEC", "use_STDIN": true, "use_STDOUT": true, - "cmd": [ "bamsormadup", {"subst":"bsmd_threads"}, "SO=coordinate", "level=0", "verbose=0", "fixmate=1", "adddupmarksupport=1", {"subst":"bs_tmpfile_flag"} ] + "cmd": [ {"subst":"bsc_executable", "required":"yes", "ifnull":"bamsormadup"}, {"subst":"bsmd_threads"}, "SO=coordinate", "level=0", "verbose=0", "fixmate=1", "adddupmarksupport=1", {"subst":"bs_tmpfile_flag"} ] }, { "id":"bammarkduplicates", From 5f0d8fc47f10a0fec7c3246570a998b56f131847 Mon Sep 17 00:00:00 2001 From: Carol Scott Date: Tue, 3 Oct 2017 16:11:09 +0100 Subject: [PATCH 14/20] Allow scramble to have optional embed reference param in final_output_prep --- Changes | 2 +- data/vtlib/final_output_prep.json | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Changes b/Changes index 4c81d362c..0fa1a4a04 100644 --- a/Changes +++ b/Changes @@ -1,7 +1,7 @@ CHANGES LOG ----------- - - Small change in final_output_prep to allow bamsort_cmd to have configurable executable. + - Small change in final_output_prep to allow bamsort_cmd to have configurable executable and scramble to have optional embed reference param. release 0.18.6 - Y-split fixes diff --git a/data/vtlib/final_output_prep.json b/data/vtlib/final_output_prep.json index 18765920d..8222476d7 100644 --- a/data/vtlib/final_output_prep.json +++ b/data/vtlib/final_output_prep.json @@ -268,7 +268,14 @@ {"subst":"b2c_compress_level", "ifnull":"-7"}, "-I", "bam", "-O", "cram", - {"subst":"scramble_reference_flag"} ] + {"subst":"scramble_reference_flag"}, + {"select":"scramble_embed_reference","default":0,"select_range":[0,1], + "cases":[ + [], + "-e" + ] + } + ] }, { "id":"scramble_tee", From 1c457b3945287d38499ae130c8fb460945c95769 Mon Sep 17 00:00:00 2001 From: Kevin Lewis Date: Wed, 4 Oct 2017 12:04:33 +0100 Subject: [PATCH 15/20] =?UTF-8?q?correctly=20distinguish=20between=20empty?= =?UTF-8?q?=20and=20undefined=20default=20for=20select=20=E2=80=A6=20(#190?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * correctly distinguish between empty and undefined default for select directive * add tests --- bin/vtfp.pl | 2 +- t/10-vtfp-select_directive.t | 68 ++++++++++++++++++++++++++++++++- t/10-vtfp-select_directive.v2.t | 68 ++++++++++++++++++++++++++++++++- 3 files changed, 135 insertions(+), 3 deletions(-) diff --git a/bin/vtfp.pl b/bin/vtfp.pl index 88fb825d9..2a98e3290 100755 --- a/bin/vtfp.pl +++ b/bin/vtfp.pl @@ -833,7 +833,7 @@ sub resolve_select_value { $default = subst_walk($select->{default}, $params, $ewi, $aux); } - if(not $default or (ref $default eq q[ARRAY] and not @{$default})) { return; } + if(not defined $default or (ref $default eq q[ARRAY] and not @{$default})) { return; } $indexes = $default; if(not ref $indexes) { $indexes = [ $indexes ]; } diff --git a/t/10-vtfp-select_directive.t b/t/10-vtfp-select_directive.t index f7a6cd2b6..2b4403234 100644 --- a/t/10-vtfp-select_directive.t +++ b/t/10-vtfp-select_directive.t @@ -1,7 +1,7 @@ use strict; use warnings; use Carp; -use Test::More tests => 6; +use Test::More tests => 7; use Test::Cmd; use File::Slurp; use Perl6::Slurp; @@ -903,4 +903,70 @@ subtest 'select_directive_no_sel' => sub { is_deeply ($vtfp_results, $expected_result, 'select no values from cases (hash), overriding default with nullkeys'); }; +### confirm distinction between 0 and undef as default value in a select directive +subtest 'select_directive_no_sel' => sub { + plan tests => 4; + + # with array cases + my $select_cmd_template = { + version => "1.0", + description => "select multiple values from cases (array)", + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {select => "wordsel", "default" => 0, cases => [ "one", "two", "three", "four" ]} ] + }, + ], + }; + + my $template = $tdir.q[/10-vtfp-select_cmd_5_0.json]; + my $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '1.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "one" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select value using default from cases (array)'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -nullkeys wordsel $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '1.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'forcing undef value in select directive'); +}; + 1; diff --git a/t/10-vtfp-select_directive.v2.t b/t/10-vtfp-select_directive.v2.t index cb1cac3e9..1fb0bfa55 100644 --- a/t/10-vtfp-select_directive.v2.t +++ b/t/10-vtfp-select_directive.v2.t @@ -1,7 +1,7 @@ use strict; use warnings; use Carp; -use Test::More tests => 6; +use Test::More tests => 7; use Test::Cmd; use File::Slurp; use Perl6::Slurp; @@ -903,4 +903,70 @@ subtest 'select_directive_no_sel' => sub { is_deeply ($vtfp_results, $expected_result, 'select no values from cases (hash), overriding default with nullkeys'); }; +### confirm distinction between 0 and undef as default value in a select directive +subtest 'select_directive_no_sel' => sub { + plan tests => 4; + + # with array cases + my $select_cmd_template = { + version => "2.0", + description => "select multiple values from cases (array)", + nodes => [ + { + id => "A", + type => "EXEC", + use_STDIN => JSON::false, + use_STDOUT => JSON::true, + cmd => [ "echo", {select => "wordsel", "default" => 0, cases => [ "one", "two", "three", "four" ]} ] + }, + ], + }; + + my $template = $tdir.q[/10-vtfp-select_cmd_5_0.json]; + my $template_contents = to_json($select_cmd_template); + write_file($template, $template_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo", "one" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'select value using default from cases (array)'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -nullkeys wordsel $template]); + ok($exit_status>>8 == 0, "zero exit for test run: $exit_status"); + $vtfp_results = from_json($test->stdout); + + $expected_result = { + version => '2.0', + nodes => [ + { + type => "EXEC", + use_STDOUT => JSON::true, + cmd => [ "echo" ], + use_STDIN => JSON::false, + id => "A" + }, + ], + edges => [ + ], + }; + + is_deeply ($vtfp_results, $expected_result, 'forcing undef value in select directive'); +}; + 1; From 91989860c2bfb6237ced3d0add22e223408c5842 Mon Sep 17 00:00:00 2001 From: Kevin Lewis Date: Thu, 5 Oct 2017 09:37:51 +0100 Subject: [PATCH 16/20] Full path node names (#191) * nodes in subgraphs (VTFILE nodes) should be prefixed by node_prefixes from all ancestor templates to ensure unique id values for all nodes in final flat graph * amend tests to handle full nodes names in subgraphs --- bin/vtfp.pl | 46 +++++++++++++++++------------------------- t/10-vtfp-vtfile.mod.t | 20 +++++++++--------- t/10-vtfp-vtfile.t | 20 +++++++++--------- t/10-vtfp-vtfile_v2.t | 20 +++++++++--------- 4 files changed, 48 insertions(+), 58 deletions(-) diff --git a/bin/vtfp.pl b/bin/vtfp.pl index 2a98e3290..ac07c22a4 100755 --- a/bin/vtfp.pl +++ b/bin/vtfp.pl @@ -1020,17 +1020,20 @@ sub report_pv_ewi { # the resulting graph with one of the visualisation tools.) ####################################################################################### sub flatten_tree { - my ($tree_node, $tver_default, $flat_graph) = @_; + my ($tree_node, $tver_default, $flat_graph, $ancestor_prefixes) = @_; $flat_graph ||= {}; + $ancestor_prefixes ||= []; # insert edges and nodes from current tree_node to $flat_graph - subgraph_to_flat_graph($tree_node, $tver_default, $flat_graph); + subgraph_to_flat_graph($tree_node, $tver_default, $flat_graph, $ancestor_prefixes); # do the same recursively for any children + push @{$ancestor_prefixes}, ($tree_node->{node_prefix} || q[]); for my $tn (@{$tree_node->{children}}) { - flatten_tree($tn, $tver_default, $flat_graph); + flatten_tree($tn, $tver_default, $flat_graph, $ancestor_prefixes); } + pop @{$ancestor_prefixes}; return $flat_graph; } @@ -1040,29 +1043,27 @@ sub flatten_tree { # losing everything except nodes and edges is a possibly undesirable side-effect of this ######################################################################################### sub subgraph_to_flat_graph { - my ($tree_node, $tver_default, $flat_graph) = @_; + my ($tree_node, $tver_default, $flat_graph, $ancestor_prefixes) = @_; my $vtnode_id = $tree_node->{id}; my $vt_name = $tree_node->{name}; my $subcfg = $tree_node->{cfg}; + my $ancestor_prefix = join(q//, @{$ancestor_prefixes}); + ################################################################################### # prefix the nodes in this subgraph with a prefix to ensure uniqueness of id values ################################################################################### my $tver = ($subcfg->{version} or $tver_default); - $subcfg->{nodes} = [ (map { $_->{id} = sprintf "%s%s", $tree_node->{node_prefix}, $_->{id}; if($_->{type} eq q[EXEC] and not $_->{tver} and $tver ne $tver_default) { $_->{tver} = $tver; } $_; } @{$subcfg->{nodes}}) ]; + $subcfg->{nodes} = [ (map { $_->{id} = sprintf "%s%s%s", $ancestor_prefix, $tree_node->{node_prefix}, $_->{id}; if($_->{type} eq q[EXEC] and not $_->{tver} and $tver ne $tver_default) { $_->{tver} = $tver; } $_; } @{$subcfg->{nodes}}) ]; ######################################################################## # any edges which refer to nodes in this subgraph should also be updated ######################################################################## for my $edge (@{$subcfg->{edges}}) { - if(not get_child_prefix($tree_node->{children}, $edge->{from})) { # if there is a child prefix, this belongs to a subgraph - don't prefix it - $edge->{from} = sprintf "%s%s", $tree_node->{node_prefix}, $edge->{from}; - } - if(not get_child_prefix($tree_node->{children}, $edge->{to})) { # if there is a child prefix, this belongs to a subgraph - don't prefix it - $edge->{to} = sprintf "%s%s", $tree_node->{node_prefix}, $edge->{to}; - } + $edge->{from} = sprintf "%s%s%s", $ancestor_prefix, $tree_node->{node_prefix}, $edge->{from}; + $edge->{to} = sprintf "%s%s%s", $ancestor_prefix, $tree_node->{node_prefix}, $edge->{to}; } ########################################################## @@ -1078,10 +1079,10 @@ sub subgraph_to_flat_graph { # now fiddle the edges in the flattened graph (maybe "fiddle" should be defined) # first inputs to the subgraph... (identify edges in the flat graph which terminate in nodes of this subgraph; use the subgraph_io section of the subgraph to remap these edge destinations) - my $in_edges = [ (grep { $_->{to} =~ /^$vtnode_id(:|$)/; } @{$flat_graph->{edges}}) ]; + my $in_edges = [ (grep { $_->{to} =~ /^$ancestor_prefix$vtnode_id(:|$)/; } @{$flat_graph->{edges}}) ]; if(@$in_edges and not $subgraph_nodes_in) { $logger->($VLFATAL, q[Cannot remap VTFILE node "], $vtnode_id, q[". No inputs specified in subgraph ], $vt_name); } for my $edge (@$in_edges) { - if($edge->{to} =~ /^$vtnode_id:?(.*)$/) { + if($edge->{to} =~ /^$ancestor_prefix$vtnode_id:?(.*)$/) { my $portkey = $1; $portkey ||= q[_stdin_]; @@ -1109,12 +1110,7 @@ sub subgraph_to_flat_graph { else { $mod_edge = $edge; } - if(get_child_prefix($tree_node->{children}, $ports->[$i])) { # if there is a child prefix, this belongs to a subgraph - don't prefix it - $mod_edge->{to} = $ports->[$i]; - } - else { - $mod_edge->{to} = sprintf "%s%s", $tree_node->{node_prefix}, $ports->[$i]; - } + $mod_edge->{to} = sprintf "%s%s%s", $ancestor_prefix, $tree_node->{node_prefix}, $ports->[$i]; } } else { @@ -1124,10 +1120,10 @@ sub subgraph_to_flat_graph { } # ...then outputs from the subgraph (identify edges in the flat graph which originate in nodes of the subgraph; use the subgraph_io section of the subgraph to remap these edge destinations) - my $out_edges = [ (grep { $_->{from} =~ /^$vtnode_id(:|$)/; } @{$flat_graph->{edges}}) ]; + my $out_edges = [ (grep { $_->{from} =~ /^$ancestor_prefix$vtnode_id(:|$)/; } @{$flat_graph->{edges}}) ]; if(@$out_edges and not $subgraph_nodes_out) { $logger->($VLFATAL, q[Cannot remap VTFILE node "], $vtnode_id, q[". No outputs specified in subgraph ], $vt_name); } for my $edge (@$out_edges) { - if($edge->{from} =~ /^$vtnode_id:?(.*)$/) { + if($edge->{from} =~ /^$ancestor_prefix$vtnode_id:?(.*)$/) { my $portkey = $1; $portkey ||= q[_stdout_]; @@ -1136,13 +1132,7 @@ sub subgraph_to_flat_graph { $logger->($VLFATAL, q[Failed to map port in subgraph: ], $vtnode_id, q[:], $portkey); } - # do check for existence of port in - if(get_child_prefix($tree_node->{children}, $port)) { # if there is a child prefix, this belongs to a subgraph - don't prefix it - $edge->{from} = $port; - } - else { - $edge->{from} = sprintf "%s%s", $tree_node->{node_prefix}, $port; - } + $edge->{from} = sprintf "%s%s%s", $ancestor_prefix, $tree_node->{node_prefix}, $port; } else { $logger->($VLMIN, q[Currently only edges to stdin processed when remapping VTFILE edges. Not processing: ], $edge->{to}, q[ in edge: ], $edge->{id}); diff --git a/t/10-vtfp-vtfile.mod.t b/t/10-vtfp-vtfile.mod.t index a1be74ebb..0d61672f8 100644 --- a/t/10-vtfp-vtfile.mod.t +++ b/t/10-vtfp-vtfile.mod.t @@ -244,32 +244,32 @@ subtest 'multilevel_vtf' => sub { cmd => [ 'tee', '__A_OUT__', '__B_OUT__' ] }, { - id => 'aout_rev', + id => 'vtf1_aout_rev', type => 'EXEC', cmd => [ 'rev' ] }, { - id => 'aout_file', + id => 'vtf1_aout_file', type => 'OUTFILE', name => 'tmp.xxx' }, { - id => 'bout_rev', + id => 'vtf1_bout_rev', type => 'EXEC', cmd => [ 'rev' ] }, { - id => 'bout_file', + id => 'vtf1_bout_file', type => 'OUTFILE', name => 'tmp.yyy' } ], edges=> [ { id => 'e1', from => 'n1', to => 'vtf1_tee'}, - { id => 'e3', from => 'vtf1_tee:__A_OUT__', to => 'aout_rev'}, - { id => 'e4', from => 'vtf1_tee:__B_OUT__', to => 'bout_rev'}, - { id => 'e2', from => 'aout_rev', to => 'aout_file'}, - { id => 'e2', from => 'bout_rev', to => 'bout_file'} + { id => 'e3', from => 'vtf1_tee:__A_OUT__', to => 'vtf1_aout_rev'}, + { id => 'e4', from => 'vtf1_tee:__B_OUT__', to => 'vtf1_bout_rev'}, + { id => 'e2', from => 'vtf1_aout_rev', to => 'vtf1_aout_file'}, + { id => 'e2', from => 'vtf1_bout_rev', to => 'vtf1_bout_file'} ] }; @@ -394,7 +394,7 @@ subtest 'multilevel_local_param_reeval' => sub { name => 'tmp.wxyz', }, { - id => 'vtf12_vfile', + id => 'vtf11_vtf12_vfile', type => 'OUTFILE', name => 'tmp.weez', }, @@ -402,7 +402,7 @@ subtest 'multilevel_local_param_reeval' => sub { edges=> [ { id => 'e1', from => 'n1', to => 'vtf11_tee'}, { id => 'e2', from => 'vtf11_tee:__A_OUT__', to => 'vtf11_file'}, - { id => 'e3', from => 'vtf11_tee:__B_OUT__', to => 'vtf12_vfile'} + { id => 'e3', from => 'vtf11_tee:__B_OUT__', to => 'vtf11_vtf12_vfile'} ] }; diff --git a/t/10-vtfp-vtfile.t b/t/10-vtfp-vtfile.t index a1be74ebb..0d61672f8 100644 --- a/t/10-vtfp-vtfile.t +++ b/t/10-vtfp-vtfile.t @@ -244,32 +244,32 @@ subtest 'multilevel_vtf' => sub { cmd => [ 'tee', '__A_OUT__', '__B_OUT__' ] }, { - id => 'aout_rev', + id => 'vtf1_aout_rev', type => 'EXEC', cmd => [ 'rev' ] }, { - id => 'aout_file', + id => 'vtf1_aout_file', type => 'OUTFILE', name => 'tmp.xxx' }, { - id => 'bout_rev', + id => 'vtf1_bout_rev', type => 'EXEC', cmd => [ 'rev' ] }, { - id => 'bout_file', + id => 'vtf1_bout_file', type => 'OUTFILE', name => 'tmp.yyy' } ], edges=> [ { id => 'e1', from => 'n1', to => 'vtf1_tee'}, - { id => 'e3', from => 'vtf1_tee:__A_OUT__', to => 'aout_rev'}, - { id => 'e4', from => 'vtf1_tee:__B_OUT__', to => 'bout_rev'}, - { id => 'e2', from => 'aout_rev', to => 'aout_file'}, - { id => 'e2', from => 'bout_rev', to => 'bout_file'} + { id => 'e3', from => 'vtf1_tee:__A_OUT__', to => 'vtf1_aout_rev'}, + { id => 'e4', from => 'vtf1_tee:__B_OUT__', to => 'vtf1_bout_rev'}, + { id => 'e2', from => 'vtf1_aout_rev', to => 'vtf1_aout_file'}, + { id => 'e2', from => 'vtf1_bout_rev', to => 'vtf1_bout_file'} ] }; @@ -394,7 +394,7 @@ subtest 'multilevel_local_param_reeval' => sub { name => 'tmp.wxyz', }, { - id => 'vtf12_vfile', + id => 'vtf11_vtf12_vfile', type => 'OUTFILE', name => 'tmp.weez', }, @@ -402,7 +402,7 @@ subtest 'multilevel_local_param_reeval' => sub { edges=> [ { id => 'e1', from => 'n1', to => 'vtf11_tee'}, { id => 'e2', from => 'vtf11_tee:__A_OUT__', to => 'vtf11_file'}, - { id => 'e3', from => 'vtf11_tee:__B_OUT__', to => 'vtf12_vfile'} + { id => 'e3', from => 'vtf11_tee:__B_OUT__', to => 'vtf11_vtf12_vfile'} ] }; diff --git a/t/10-vtfp-vtfile_v2.t b/t/10-vtfp-vtfile_v2.t index a60e154f8..f642f8572 100644 --- a/t/10-vtfp-vtfile_v2.t +++ b/t/10-vtfp-vtfile_v2.t @@ -244,32 +244,32 @@ subtest 'multilevel_vtf' => sub { cmd => [ 'tee', {'port' => 'a'} , {'port' => 'b'}, ] }, { - id => 'aout_rev', + id => 'vtf1_aout_rev', type => 'EXEC', cmd => [ 'rev' ] }, { - id => 'aout_file', + id => 'vtf1_aout_file', type => 'OUTFILE', name => 'tmp.xxx' }, { - id => 'bout_rev', + id => 'vtf1_bout_rev', type => 'EXEC', cmd => [ 'rev' ] }, { - id => 'bout_file', + id => 'vtf1_bout_file', type => 'OUTFILE', name => 'tmp.yyy' } ], edges=> [ { id => 'e1', from => 'n1', to => 'vtf1_tee'}, - { id => 'e3', from => 'vtf1_tee:a', to => 'aout_rev'}, - { id => 'e4', from => 'vtf1_tee:b', to => 'bout_rev'}, - { id => 'e2', from => 'aout_rev', to => 'aout_file'}, - { id => 'e2', from => 'bout_rev', to => 'bout_file'} + { id => 'e3', from => 'vtf1_tee:a', to => 'vtf1_aout_rev'}, + { id => 'e4', from => 'vtf1_tee:b', to => 'vtf1_bout_rev'}, + { id => 'e2', from => 'vtf1_aout_rev', to => 'vtf1_aout_file'}, + { id => 'e2', from => 'vtf1_bout_rev', to => 'vtf1_bout_file'} ] }; @@ -394,7 +394,7 @@ subtest 'multilevel_local_param_reeval' => sub { name => 'tmp.wxyz', }, { - id => 'vtf12_vfile', + id => 'vtf11_vtf12_vfile', type => 'OUTFILE', name => 'tmp.weez', }, @@ -402,7 +402,7 @@ subtest 'multilevel_local_param_reeval' => sub { edges=> [ { id => 'e1', from => 'n1', to => 'vtf11_tee'}, { id => 'e2', from => 'vtf11_tee:__A_OUT__', to => 'vtf11_file'}, - { id => 'e3', from => 'vtf11_tee:__B_OUT__', to => 'vtf12_vfile'} + { id => 'e3', from => 'vtf11_tee:__B_OUT__', to => 'vtf11_vtf12_vfile'} ] }; From a70655fed59b033c6675fbc1d2d3d19b350ed712 Mon Sep 17 00:00:00 2001 From: "Ruben E. Bautista" Date: Wed, 25 Oct 2017 16:55:26 +0100 Subject: [PATCH 17/20] Drop alignments to transcriptome and split of chimeric reads - into separate bam files, and thus too their conversions to cram format - chimeric files are being included within the main alignments output - update the changelog - update manifest file to include salmon and star templates --- Changes | 2 + MANIFEST | 2 + data/vtlib/star_alignment.json | 106 +-------------------------------- 3 files changed, 6 insertions(+), 104 deletions(-) diff --git a/Changes b/Changes index 36fc50ce2..9f06da452 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,8 @@ CHANGES LOG ----------- + - add new templates for STAR alignment and Salmon + release 0.18.6 - Y-split fixes When processing VTFILE nodes, make sure that parameters are reevaluated locally instead of naively inheriting values diff --git a/MANIFEST b/MANIFEST index 47711cac9..f3270c406 100644 --- a/MANIFEST +++ b/MANIFEST @@ -25,10 +25,12 @@ data/vtlib/pre_alignment.json data/vtlib/pre_alignment_realign.json data/vtlib/README.vtlib data/vtlib/realignment_wtsi_template.json +data/vtlib/salmon_alignment.json data/vtlib/seqchksum.json data/vtlib/seqchksum_hs.json data/vtlib/seqchksum_realign.json data/vtlib/split_by_chromosome.json +data/vtlib/star_alignment.json data/vtlib/tophat2_alignment.json examples/bwa_aln_cfg.png examples/bwa_mem/bwa_mem_alignment.vtf diff --git a/data/vtlib/star_alignment.json b/data/vtlib/star_alignment.json index 90defe7ef..4d59d00a2 100644 --- a/data/vtlib/star_alignment.json +++ b/data/vtlib/star_alignment.json @@ -109,22 +109,6 @@ "postproc":{"op":"concat","pad":""} } }, - { - "id":"totranscriptome_bam", - "required":"no", - "subst_constructor":{ - "vals":[ {"subst":"star_out"}, "Aligned.toTranscriptome.out.bam" ], - "postproc":{"op":"concat","pad":""} - } - }, - { - "id":"chimeric_sam", - "required":"no", - "subst_constructor":{ - "vals":[ {"subst":"star_out"}, "Chimeric.out.sam" ], - "postproc":{"op":"concat","pad":""} - } - }, { "id":"readspergene_tab", "required":"no", @@ -133,42 +117,6 @@ "postproc":{"op":"concat","pad":""} } }, - { - "id":"scramble_chimeric_sam_target", - "required":"no", - "subst_constructor":{ - "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".chimeric.cram" ], - "postproc":{"op":"concat","pad":""} - } - }, - { - "id":"scramble_totranscriptome_bam_target", - "required":"no", - "subst_constructor":{ - "vals":[ {"subst":"outdatadir"}, "/", {"subst":"rpt"}, ".totranscriptome.cram" ], - "postproc":{"op":"concat","pad":""} - } - }, - { - "id":"reference_genome_fasta_name","required":"no"}, - { - "id":"reference_genome_fasta", - "required":"yes", - "subst_constructor":{ - "vals":[ {"subst":"reposdir"}, "/", {"subst":"reference_genome_fasta_name"} ], - "postproc":{"op":"concat", "pad":""} - } - }, - { - "id":"reference_transcriptome_fasta_name","required":"no"}, - { - "id":"reference_transcriptome_fasta", - "required":"yes", - "subst_constructor":{ - "vals":[ {"subst":"reposdir"}, "/", {"subst":"reference_transcriptome_fasta_name"} ], - "postproc":{"op":"concat", "pad":""} - } - }, { "id":"cp_junctions_tab_target", "required":"no", @@ -235,10 +183,10 @@ "--outSAMunmapped", "Within", "KeepPairs", "--outSAMtype", "BAM", "Unsorted", "--outFilterIntronMotifs", "RemoveNoncanonicalUnannotated", - "--chimOutType", "SeparateSAMold", + "--chimOutType", "WithinBAM", "--chimSegmentMin", "15", "--chimJunctionOverhangMin", "15", - "--quantMode", "TranscriptomeSAM", "GeneCounts", + "--quantMode", "GeneCounts", "--genomeDir", "__REFERENCE_GENOME_IN__", "--readFilesIn", "__FQ1_IN__", "__FQ2_IN__", "--outStd", "BAM_Unsorted" @@ -250,58 +198,12 @@ "subtype":"DUMMY", "name":{"subst":"junctions_tab"} }, - { - "id":"totranscriptome_bam", - "type":"RAFILE", - "subtype":"DUMMY", - "name":{"subst":"totranscriptome_bam"} - }, - { - "id":"chimeric_sam", - "type":"RAFILE", - "subtype":"DUMMY", - "name":{"subst":"chimeric_sam"} - }, { "id":"readspergene_tab", "type":"RAFILE", "subtype":"DUMMY", "name":{"subst":"readspergene_tab"} }, - { - "id":"scramble_chimeric_sam", - "type":"EXEC", - "use_STDIN": false, - "use_STDOUT": false, - "cmd":[ - "scramble", - {"subst":"b2c_mt", "ifnull":{"subst_constructor":{ "vals":[ "-t", {"subst":"b2c_mt_val"} ]}}}, - {"subst":"b2c_fmtver", "ifnull":{"subst_constructor":{ "vals":[ "-V", {"subst":"b2c_format_version"} ]}}}, - {"subst":"b2c_compress_level", "ifnull":"-7"}, - "-I", "sam", - "-O", "cram", - "-r", {"subst":"reference_genome_fasta"}, - "__SRC_CHIMERIC_SAM_IN__", - {"subst":"scramble_chimeric_sam_target"} - ] - }, - { - "id":"scramble_totranscriptome_bam", - "type":"EXEC", - "use_STDIN": false, - "use_STDOUT": false, - "cmd":[ - "scramble", - {"subst":"b2c_mt", "ifnull":{"subst_constructor":{ "vals":[ "-t", {"subst":"b2c_mt_val"} ]}}}, - {"subst":"b2c_fmtver", "ifnull":{"subst_constructor":{ "vals":[ "-V", {"subst":"b2c_format_version"} ]}}}, - {"subst":"b2c_compress_level", "ifnull":"-7"}, - "-I", "bam", - "-O", "cram", - "-r", {"subst":"reference_transcriptome_fasta"}, - "__SRC_TOTRANSCRIPTOME_BAM_IN__", - {"subst":"scramble_totranscriptome_bam_target"} - ] - }, { "id":"cp_junctions_tab", "type":"EXEC", @@ -332,10 +234,6 @@ { "id":"bamtofastq_to_fq2", "from":"bamtofastq:__FQ2_OUT__", "to":"fq2" }, { "id":"fq1_to_star", "from":"fq1", "to":"star:__FQ1_IN__" }, { "id":"fq2_to_star", "from":"fq2", "to":"star:__FQ2_IN__" }, - { "id":"star_to_chimeric_sam", "from":"star", "to":"chimeric_sam" }, - { "id":"scramble_chimeric_sam", "from":"chimeric_sam", "to":"scramble_chimeric_sam:__SRC_CHIMERIC_SAM_IN__"}, - { "id":"star_to_totranscriptome_bam", "from":"star", "to":"totranscriptome_bam" }, - { "id":"scramble_totranscriptome_bam", "from":"totranscriptome_bam", "to":"scramble_totranscriptome_bam:__SRC_TOTRANSCRIPTOME_BAM_IN__" }, { "id":"star_to_junctions_tab", "from":"star", "to":"junctions_tab" }, { "id":"cp_junctions_tab", "from":"junctions_tab", "to":"cp_junctions_tab:__SRC_JUNCTIONS_TAB_IN__" }, { "id":"star_to_readspergene_tab", "from":"star", "to":"readspergene_tab" }, From 7ab36d23f4b4e1f7c0f40960ecf37bcb2e9f6c43 Mon Sep 17 00:00:00 2001 From: "Ruben E. Bautista" Date: Thu, 9 Nov 2017 11:27:17 +0000 Subject: [PATCH 18/20] Add subst_param flags for chimeric options instead of hard-coded values - set ifnull attributes to defaults defined in STAR documentation --- data/vtlib/star_alignment.json | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/data/vtlib/star_alignment.json b/data/vtlib/star_alignment.json index 4d59d00a2..160ad6fd3 100644 --- a/data/vtlib/star_alignment.json +++ b/data/vtlib/star_alignment.json @@ -101,6 +101,32 @@ "postproc":{"op":"concat","pad":" "} } }, + { + "id":"chimSegmentMin_flag", + "required":"no", + "subst_constructor":{ + "vals":[ "--chimSegmentMin", { + "subst":"chimSegmentMin_val", + "inull":"0", + "comment":"unset this value to remove --chimSegmentMin flag" + } + ], + "postproc":{"op":"concat","pad":" "} + } + }, + { + "id":"chimJunctionOverhangMin_flag", + "required":"no", + "subst_constructor":{ + "vals":[ "--chimJunctionOverhangMin", { + "subst":"chimJunctionOverhangMin_val", + "ifnull":"20", + "comment":"unset this value to remove --chimJunctionOverhangMin flag" + } + ], + "postproc":{"op":"concat","pad":" "} + } + }, { "id":"junctions_tab", "required":"no", @@ -184,8 +210,8 @@ "--outSAMtype", "BAM", "Unsorted", "--outFilterIntronMotifs", "RemoveNoncanonicalUnannotated", "--chimOutType", "WithinBAM", - "--chimSegmentMin", "15", - "--chimJunctionOverhangMin", "15", + {"subst":"chimSegmentMin_flag"}, + {"subst":"chimJunctionOverhangMin_flag"}, "--quantMode", "GeneCounts", "--genomeDir", "__REFERENCE_GENOME_IN__", "--readFilesIn", "__FQ1_IN__", "__FQ2_IN__", From 485833de8a16b4a34a90ad513be780d8bb55c1f5 Mon Sep 17 00:00:00 2001 From: "Ruben E. Bautista" Date: Thu, 16 Nov 2017 15:24:24 +0000 Subject: [PATCH 19/20] Fix typo --- data/vtlib/star_alignment.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/vtlib/star_alignment.json b/data/vtlib/star_alignment.json index 160ad6fd3..55bd6cad1 100644 --- a/data/vtlib/star_alignment.json +++ b/data/vtlib/star_alignment.json @@ -107,7 +107,7 @@ "subst_constructor":{ "vals":[ "--chimSegmentMin", { "subst":"chimSegmentMin_val", - "inull":"0", + "ifnull":"0", "comment":"unset this value to remove --chimSegmentMin flag" } ], From 627dbb9e59677c6951ff710a2b26574ac10ee74c Mon Sep 17 00:00:00 2001 From: Kevin Lewis Date: Mon, 20 Nov 2017 16:52:29 +0000 Subject: [PATCH 20/20] correct failure to report errors lower than one VTFILE level (#193) add tests remove duplicate test file --- bin/vtfp.pl | 4 +- t/10-vtfp-vtfile.mod.t | 412 ----------------------------------------- t/10-vtfp-vtfile.t | 137 +++++++++++++- t/10-vtfp-vtfile_v2.t | 137 +++++++++++++- 4 files changed, 274 insertions(+), 416 deletions(-) delete mode 100644 t/10-vtfp-vtfile.mod.t diff --git a/bin/vtfp.pl b/bin/vtfp.pl index ac07c22a4..0dad79bcc 100755 --- a/bin/vtfp.pl +++ b/bin/vtfp.pl @@ -1001,10 +1001,10 @@ sub report_pv_ewi { # do the same recursively for any children for my $tn (@{$tree_node->{children}}) { - if($tn->{ewi}->{report}->(0, $logger, $cull_node_ids)) { $fatality = 1; } + if(report_pv_ewi($tn, $logger, $cull_node_ids)) { $fatality = 1; } } - return $fatality; # should return some kind of error indicator, I think + return $fatality; } ####################################################################################### diff --git a/t/10-vtfp-vtfile.mod.t b/t/10-vtfp-vtfile.mod.t deleted file mode 100644 index 0d61672f8..000000000 --- a/t/10-vtfp-vtfile.mod.t +++ /dev/null @@ -1,412 +0,0 @@ -use strict; -use warnings; -use Carp; -use Test::More tests => 4; -use Test::Cmd; -use File::Slurp; -use Perl6::Slurp; -use JSON; -use File::Temp qw(tempdir); -use Cwd; - -my $tdir = tempdir(CLEANUP => 1); - -my $odir = getcwd(); -my $test = Test::Cmd->new( prog => $odir.'/bin/vtfp.pl', workdir => q()); -ok($test, 'made test object'); - -# Test VTFILE processing by vtfp - -# basic functions -subtest 'basic_checks' => sub { - plan tests => 2; - - my $basic_container = { - description => 'basic template containing a VTFILE node', - version => '1.0', - nodes => [ - { - id => 'n1', - type => 'EXEC', - cmd => [ 'echo', 'aeronautics'] - }, - { - id => 'v1', - type => 'VTFILE', - node_prefix => 'vtf0_', - name => "$tdir/10-vtfp-vtfile_vtf0.json" - } - ], - edges => [ - { id => 'e1', from => 'n1', to => 'v1'} - ] - }; - - my $vtf0 = { - description => 'basic VTFILE', - version => '1.0', - subgraph_io => { - ports => { - inputs => { - _stdin_ => 'vowelrot' - } - } - }, - nodes => [ - { - id => 'vowelrot', - type => 'EXEC', - cmd => [ 'tr', 'aeiou', 'eioua' ] - } - ] - }; - - my $template = $tdir.q[/10-vtfp-vtfile_basic.json]; - my $template_contents = to_json($basic_container); - write_file($template, $template_contents); - - my $vtfile = $tdir.q[/10-vtfp-vtfile_vtf0.json]; - my $vtfile_contents = to_json($vtf0); - write_file($vtfile, $vtfile_contents); - - my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); - ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); - my $vtfp_results = from_json($test->stdout); - - my $expected_result = { - version => '1.0', - nodes => [ - { - id => 'n1', - type => 'EXEC', - cmd => ['echo', 'aeronautics'] - }, - { - id => 'vtf0_vowelrot', - type => 'EXEC', - cmd => [ 'tr', 'aeiou', 'eioua' ] - } - ], - edges=> [ - { id => 'e1', from => 'n1', to => 'vtf0_vowelrot'} - ] - }; - - is_deeply ($vtfp_results, $expected_result, 'basic check'); -}; - -subtest 'multilevel_vtf' => sub { - plan tests => 4; - - my $basic_container = { - description => 'basic template containing a VTFILE node', - version => '1.0', - nodes => [ - { - id => 'n1', - type => 'EXEC', - cmd => [ 'echo', 'aeronautics'] - }, - { - id => 'v1', - type => 'VTFILE', - node_prefix => 'vtf1_', - name => { subst => 'vtfname', ifnull => "$tdir/10-vtfp-vtfile_vtf1.json" } - } - ], - edges => [ - { id => 'e1', from => 'n1', to => 'v1'} - ] - }; - - my $vtf1 = { - description => 'unary', - version => '1.0', - subgraph_io => { - ports => { - inputs => { - _stdin_ => 'rev', - } - } - }, - nodes => [ - { - id => 'rev', - type => 'EXEC', - cmd => [ 'rev' ] - }, - { - id => 'file', - type => 'OUTFILE', - name => { subst_constructor => { vals => [ 'tmp.', {subst => 'ext', ifnull => 'txt', }], postproc => { op => 'concat', pad => ''} }, } - } - ], - edges => [ - { id => 'e2', from => 'rev', to => 'file'} - ] - }; - - my $vtf2 = { - description => 'binary', - version => '1.0', - subgraph_io => { - ports => { - inputs => { - _stdin_ => 'tee', - } - } - }, - nodes => [ - { - id => 'tee', - type => 'EXEC', - cmd => [ 'tee', '__A_OUT__', '__B_OUT__' ] - }, - { - id => 'aout', - type => 'VTFILE', - name => "$tdir/10-vtfp-vtfile_vtf1.json", - node_prefix => 'aout_', - subst_map => { ext =>'xxx' }, - }, - { - id => 'bout', - type => 'VTFILE', - name => "$tdir/10-vtfp-vtfile_vtf1.json", - node_prefix => 'bout_', - subst_map => { ext => 'yyy' }, - }, - ], - edges => [ - { id => 'e3', from => 'tee:__A_OUT__', to => 'aout'}, - { id => 'e4', from => 'tee:__B_OUT__', to => 'bout'}, - ] - }; - - my $template = $tdir.q[/10-vtfp-vtfile_multilevel0.json]; - my $template_contents = to_json($basic_container); - write_file($template, $template_contents); - - my $vtfile1 = $tdir.q[/10-vtfp-vtfile_vtf1.json]; - my $vtfile_contents = to_json($vtf1); - write_file($vtfile1, $vtfile_contents); - - my $vtfile2 = $tdir.q[/10-vtfp-vtfile_vtf2.json]; - $vtfile_contents = to_json($vtf2); - write_file($vtfile2, $vtfile_contents); - - my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); - ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); - my $vtfp_results = from_json($test->stdout); - - my $expected_result = { - version => '1.0', - nodes => [ - { - id => 'n1', - type => 'EXEC', - cmd => ['echo', 'aeronautics'] - }, - { - id => 'vtf1_rev', - type => 'EXEC', - cmd => [ 'rev' ] - }, - { - id => 'vtf1_file', - type => 'OUTFILE', - name => 'tmp.txt' - } - ], - edges=> [ - { id => 'e1', from => 'n1', to => 'vtf1_rev'}, - { id => 'e2', from => 'vtf1_rev', to => 'vtf1_file'} - ] - }; - - is_deeply ($vtfp_results, $expected_result, 'multilevel VTFILE nodes - first just one'); - - $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -keys vtfname -vals $vtfile2 -verbosity_level 0 $template]); - ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); - $vtfp_results = from_json($test->stdout); - - $expected_result = { - version => '1.0', - nodes => [ - { - id => 'n1', - type => 'EXEC', - cmd => ['echo', 'aeronautics'] - }, - { - id => 'vtf1_tee', - type => 'EXEC', - cmd => [ 'tee', '__A_OUT__', '__B_OUT__' ] - }, - { - id => 'vtf1_aout_rev', - type => 'EXEC', - cmd => [ 'rev' ] - }, - { - id => 'vtf1_aout_file', - type => 'OUTFILE', - name => 'tmp.xxx' - }, - { - id => 'vtf1_bout_rev', - type => 'EXEC', - cmd => [ 'rev' ] - }, - { - id => 'vtf1_bout_file', - type => 'OUTFILE', - name => 'tmp.yyy' - } - ], - edges=> [ - { id => 'e1', from => 'n1', to => 'vtf1_tee'}, - { id => 'e3', from => 'vtf1_tee:__A_OUT__', to => 'vtf1_aout_rev'}, - { id => 'e4', from => 'vtf1_tee:__B_OUT__', to => 'vtf1_bout_rev'}, - { id => 'e2', from => 'vtf1_aout_rev', to => 'vtf1_aout_file'}, - { id => 'e2', from => 'vtf1_bout_rev', to => 'vtf1_bout_file'} - ] - }; - - is_deeply ($vtfp_results, $expected_result, 'multilevel VTFILE nodes - two level (split)'); -}; - -subtest 'multilevel_local_param_reeval' => sub { - plan tests => 2; - - my $basic_container = { - description => 'top template containing a VTFILE node', - version => '1.0', - nodes => [ - { - id => 'n1', - type => 'EXEC', - cmd => [ 'echo', 'aeronautics'] - }, - { - id => 'v1', - type => 'VTFILE', - node_prefix => 'vtf11_', - name => "$tdir/10-vtfp-vtfile_vtf11.json", - subst_map => { component => 'xy' } - } - ], - edges => [ - { id => 'e1', from => 'n1', to => 'v1'} - ] - }; - - my $vtf11 = { - description => 'mid', - version => '1.0', - subgraph_io => { - ports => { - inputs => { - _stdin_ => 'tee', - } - } - }, - subst_params => [ - { id => 'ext', subst_constructor => {vals => [ 'w', {subst => 'component'}, 'z' ], postproc => { op => 'concat', pad => ''}} } - ], - nodes => [ - { - id => 'tee', - type => 'EXEC', - cmd => [ 'tee', '__A_OUT__', '__B_OUT__' ] - }, - { - id => 'file', - type => 'OUTFILE', - name => { subst_constructor => { vals => [ 'tmp.', {subst => 'ext', ifnull => 'tat', }], postproc => { op => 'concat', pad => ''} }, } - }, - { - id => 'vfile', - type => 'VTFILE', - node_prefix => 'vtf12_', - name => "$tdir/10-vtfp-vtfile_vtf12.json", - subst_map => { component => 'ee' } - }, - ], - edges => [ - { id => 'e2', from => 'tee:__A_OUT__', to => 'file'}, - { id => 'e3', from => 'tee:__B_OUT__', to => 'vfile'}, - ] - }; - - my $vtf12 = { - description => 'bottom', - comment => 'the value of param ext should not be inherited from the cache of the parent, since the passed component value should force local reevaluation', - version => '1.0', - subgraph_io => { - ports => { - inputs => { - _stdin_ => 'vfile', - } - } - }, - nodes => [ - { - id => 'vfile', - type => 'OUTFILE', - name => { subst_constructor => { vals => [ 'tmp.', {subst => 'ext'} ], postproc => { op => 'concat', pad => ''} }, } - }, - ] - }; - - my $template = $tdir.q[/10-vtfp-vtfile_multilevel1.json]; - my $template_contents = to_json($basic_container); - write_file($template, $template_contents); - - my $vtfile11 = $tdir.q[/10-vtfp-vtfile_vtf11.json]; - my $vtfile_contents = to_json($vtf11); - write_file($vtfile11, $vtfile_contents); - - my $vtfile12 = $tdir.q[/10-vtfp-vtfile_vtf12.json]; - $vtfile_contents = to_json($vtf12); - write_file($vtfile12, $vtfile_contents); - - my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 $template]); - ok($exit_status>>8 == 0, "non-zero exit for test1: $exit_status"); - my $vtfp_results = from_json($test->stdout); - - my $expected_result = { - version => '1.0', - nodes => [ - { - id => 'n1', - type => 'EXEC', - cmd => ['echo', 'aeronautics'] - }, - { - id => 'vtf11_tee', - type => 'EXEC', - cmd => [ 'tee', '__A_OUT__', '__B_OUT__' ] - }, - { - id => 'vtf11_file', - type => 'OUTFILE', - name => 'tmp.wxyz', - }, - { - id => 'vtf11_vtf12_vfile', - type => 'OUTFILE', - name => 'tmp.weez', - }, - ], - edges=> [ - { id => 'e1', from => 'n1', to => 'vtf11_tee'}, - { id => 'e2', from => 'vtf11_tee:__A_OUT__', to => 'vtf11_file'}, - { id => 'e3', from => 'vtf11_tee:__B_OUT__', to => 'vtf11_vtf12_vfile'} - ] - }; - - is_deeply ($vtfp_results, $expected_result, 'multilevel local param reeval'); -}; - -1; diff --git a/t/10-vtfp-vtfile.t b/t/10-vtfp-vtfile.t index 0d61672f8..f31ff55e3 100644 --- a/t/10-vtfp-vtfile.t +++ b/t/10-vtfp-vtfile.t @@ -1,7 +1,7 @@ use strict; use warnings; use Carp; -use Test::More tests => 4; +use Test::More tests => 5; use Test::Cmd; use File::Slurp; use Perl6::Slurp; @@ -409,4 +409,139 @@ subtest 'multilevel_local_param_reeval' => sub { is_deeply ($vtfp_results, $expected_result, 'multilevel local param reeval'); }; +subtest 'multilevel_vtf_required_param' => sub { + plan tests => 4; + + my $basic_container = { + description => 'top template containing a VTFILE node', + version => '1.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => [ 'echo', 'aeronautics'] + }, + { + id => 'v1', + type => 'VTFILE', + node_prefix => 'vtf11_', + name => "$tdir/10-vtfp-vtfile_vtf11.json" + } + ], + edges => [ + { id => 'e1', from => 'n1', to => 'v1'} + ] + }; + + my $vtf11 = { + description => 'mid', + version => '1.0', + subgraph_io => { + ports => { + inputs => { + _stdin_ => 'tee', + } + } + }, + nodes => [ + { + id => 'tee', + type => 'EXEC', + cmd => [ 'tee', '__A_OUT__', '__B_OUT__' ], + }, + { + id => 'lvfile', + type => 'VTFILE', + node_prefix => 'lvtf12_', + name => "$tdir/10-vtfp-vtfile_vtf12.json", + subst_map => { ext => 'xxx' }, + }, + { + id => 'rvfile', + type => 'VTFILE', + node_prefix => 'rvtf12_', + name => "$tdir/10-vtfp-vtfile_vtf12.json", + }, + ], + edges => [ + { id => 'e2', from => 'tee:__A_OUT__', to => 'lvfile'}, + { id => 'e3', from => 'tee:__B_OUT__', to => 'rvfile'}, + ] + }; + + my $vtf12 = { + description => 'bottom', + comment => 'the value of param ext should not be inherited from the cache of the parent, since the passed component value should force local reevaluation', + version => '1.0', + subgraph_io => { + ports => { + inputs => { + _stdin_ => 'vfile', + } + } + }, + nodes => [ + { + id => 'vfile', + type => 'OUTFILE', + name => { subst_constructor => { vals => [ 'tmp.', {subst => 'ext', 'required' => 'true'} ], postproc => { op => 'concat', pad => ''} }, } + }, + ] + }; + + my $template = $tdir.q[/10-vtfp-vtfile_multilevel1.json]; + my $template_contents = to_json($basic_container); + write_file($template, $template_contents); + + my $vtfile11 = $tdir.q[/10-vtfp-vtfile_vtf11.json]; + my $vtfile_contents = to_json($vtf11); + write_file($vtfile11, $vtfile_contents); + + my $vtfile12 = $tdir.q[/10-vtfp-vtfile_vtf12.json]; + $vtfile_contents = to_json($vtf12); + write_file($vtfile12, $vtfile_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 1 $template]); + ok($exit_status>>8 == 255, "error exit for test multilevel_vtf_required_param: $exit_status"); + my $vtfp_err = $test->stderr; + like ($vtfp_err, qr/No value found for required subst \(param_name: ext\)/, 'err ms check'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys v1:rvfile:ext -vals yyy $template]); + ok($exit_status>>8 == 0, "non-zero exit: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '1.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo', 'aeronautics'] + }, + { + id => 'vtf11_tee', + type => 'EXEC', + cmd => [ 'tee', '__A_OUT__', '__B_OUT__' ] + }, + { + id => 'vtf11_lvtf12_vfile', + type => 'OUTFILE', + name => 'tmp.xxx', + }, + { + id => 'vtf11_rvtf12_vfile', + type => 'OUTFILE', + name => 'tmp.yyy', + }, + ], + edges=> [ + { id => 'e1', from => 'n1', to => 'vtf11_tee'}, + { id => 'e2', from => 'vtf11_tee:__A_OUT__', to => 'vtf11_lvtf12_vfile'}, + { id => 'e3', from => 'vtf11_tee:__B_OUT__', to => 'vtf11_rvtf12_vfile'}, + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'multilevel local param reeval'); +}; + 1; diff --git a/t/10-vtfp-vtfile_v2.t b/t/10-vtfp-vtfile_v2.t index f642f8572..b023e965b 100644 --- a/t/10-vtfp-vtfile_v2.t +++ b/t/10-vtfp-vtfile_v2.t @@ -1,7 +1,7 @@ use strict; use warnings; use Carp; -use Test::More tests => 4; +use Test::More tests => 5; use Test::Cmd; use File::Slurp; use Perl6::Slurp; @@ -409,4 +409,139 @@ subtest 'multilevel_local_param_reeval' => sub { is_deeply ($vtfp_results, $expected_result, 'multilevel local param reeval'); }; +subtest 'multilevel_vtf_required_param' => sub { + plan tests => 4; + + my $basic_container = { + description => 'top template containing a VTFILE node', + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => [ 'echo', 'aeronautics'] + }, + { + id => 'v1', + type => 'VTFILE', + node_prefix => 'vtf11_', + name => "$tdir/10-vtfp-vtfile_vtf11.json" + } + ], + edges => [ + { id => 'e1', from => 'n1', to => 'v1'} + ] + }; + + my $vtf11 = { + description => 'mid', + version => '2.0', + subgraph_io => { + ports => { + inputs => { + _stdin_ => 'tee', + } + } + }, + nodes => [ + { + id => 'tee', + type => 'EXEC', + cmd => [ 'tee', 'left_out', 'right_out' ], + }, + { + id => 'lvfile', + type => 'VTFILE', + node_prefix => 'lvtf12_', + name => "$tdir/10-vtfp-vtfile_vtf12.json", + subst_map => { ext => 'xxx' }, + }, + { + id => 'rvfile', + type => 'VTFILE', + node_prefix => 'rvtf12_', + name => "$tdir/10-vtfp-vtfile_vtf12.json", + }, + ], + edges => [ + { id => 'e2', from => 'tee:left_out', to => 'lvfile'}, + { id => 'e3', from => 'tee:right_out', to => 'rvfile'}, + ] + }; + + my $vtf12 = { + description => 'bottom', + comment => 'the value of param ext should not be inherited from the cache of the parent, since the passed component value should force local reevaluation', + version => '2.0', + subgraph_io => { + ports => { + inputs => { + _stdin_ => 'vfile', + } + } + }, + nodes => [ + { + id => 'vfile', + type => 'OUTFILE', + name => { subst_constructor => { vals => [ 'tmp.', {subst => 'ext', 'required' => 'true'} ], postproc => { op => 'concat', pad => ''} }, } + }, + ] + }; + + my $template = $tdir.q[/10-vtfp-vtfile_multilevel1.json]; + my $template_contents = to_json($basic_container); + write_file($template, $template_contents); + + my $vtfile11 = $tdir.q[/10-vtfp-vtfile_vtf11.json]; + my $vtfile_contents = to_json($vtf11); + write_file($vtfile11, $vtfile_contents); + + my $vtfile12 = $tdir.q[/10-vtfp-vtfile_vtf12.json]; + $vtfile_contents = to_json($vtf12); + write_file($vtfile12, $vtfile_contents); + + my $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 1 $template]); + ok($exit_status>>8 == 255, "error exit for test multilevel_vtf_required_param: $exit_status"); + my $vtfp_err = $test->stderr; + like ($vtfp_err, qr/No value found for required subst \(param_name: ext\)/, 'err ms check'); + + $exit_status = $test->run(chdir => $test->curdir, args => qq[-no-absolute_program_paths -verbosity_level 0 -keys v1:rvfile:ext -vals yyy $template]); + ok($exit_status>>8 == 0, "non-zero exit: $exit_status"); + my $vtfp_results = from_json($test->stdout); + + my $expected_result = { + version => '2.0', + nodes => [ + { + id => 'n1', + type => 'EXEC', + cmd => ['echo', 'aeronautics'] + }, + { + id => 'vtf11_tee', + type => 'EXEC', + cmd => [ 'tee', 'left_out', 'right_out' ] + }, + { + id => 'vtf11_lvtf12_vfile', + type => 'OUTFILE', + name => 'tmp.xxx', + }, + { + id => 'vtf11_rvtf12_vfile', + type => 'OUTFILE', + name => 'tmp.yyy', + }, + ], + edges=> [ + { id => 'e1', from => 'n1', to => 'vtf11_tee'}, + { id => 'e2', from => 'vtf11_tee:left_out', to => 'vtf11_lvtf12_vfile'}, + { id => 'e3', from => 'vtf11_tee:right_out', to => 'vtf11_rvtf12_vfile'}, + ] + }; + + is_deeply ($vtfp_results, $expected_result, 'multilevel local param reeval'); +}; + 1;