From 85f20770c1205e728740d14a3b1539a2f4d3a29d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 19 Oct 2023 10:06:14 +0200 Subject: [PATCH 1/4] Add back 1.1.0 version of Filtering1 tool Which is the same as 1.1.1 but hard to explain to users why the run form shows a different version than what is shown in the user interface. --- lib/galaxy/config/sample/tool_conf.xml.sample | 1 + tools/stats/filtering_1_1_0.xml | 103 ++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 tools/stats/filtering_1_1_0.xml diff --git a/lib/galaxy/config/sample/tool_conf.xml.sample b/lib/galaxy/config/sample/tool_conf.xml.sample index f6f060739291..f3818f88f46c 100644 --- a/lib/galaxy/config/sample/tool_conf.xml.sample +++ b/lib/galaxy/config/sample/tool_conf.xml.sample @@ -79,6 +79,7 @@
+ diff --git a/tools/stats/filtering_1_1_0.xml b/tools/stats/filtering_1_1_0.xml new file mode 100644 index 000000000000..1a20cf40c947 --- /dev/null +++ b/tools/stats/filtering_1_1_0.xml @@ -0,0 +1,103 @@ + + data on any column using simple expressions + + operation_0335 + + + python '$__tool_directory__/filtering.py' '$input' '$out_file1' '$inputs' ${input.metadata.columns} "${input.metadata.column_types}" $header_lines + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +.. class:: warningmark + +Double equal signs, ==, must be used as *"equal to"* (e.g., **c1 == 'chr22'**) + +.. class:: infomark + +**TIP:** Attempting to apply a filtering condition may throw exceptions if the data type (e.g., string, integer) in every line of the columns being filtered is not appropriate for the condition (e.g., attempting certain numerical calculations on strings). If an exception is thrown when applying the condition to a line, that line is skipped as invalid for the filter condition. The number of invalid skipped lines is documented in the resulting history item as a "Condition/data issue". + +.. class:: infomark + +**TIP:** If your data is not TAB delimited, use *Text Manipulation->Convert* + +----- + +**Syntax** + +The filter tool allows you to restrict the dataset using simple conditional statements. + +- Columns are referenced with **c** and a **number**. For example, **c1** refers to the first column of a tab-delimited file +- Make sure that multi-character operators contain no white space ( e.g., **<=** is valid while **< =** is not valid ) +- When using 'equal-to' operator **double equal sign '==' must be used** ( e.g., **c1=='chr1'** ) +- Non-numerical values must be included in single or double quotes ( e.g., **c6=='+'** ) +- Filtering condition can include logical operators, but **make sure operators are all lower case** ( e.g., **(c1!='chrX' and c1!='chrY') or not c6=='+'** ) + +----- + +**Example** + +- **c1=='chr1'** selects lines in which the first column is chr1 +- **c3-c2<100*c4** selects lines where subtracting column 3 from column 2 is less than the value of column 4 times 100 +- **len(c2.split(',')) < 4** will select lines where the second column has less than four comma separated elements +- **c2>=1** selects lines in which the value of column 2 is greater than or equal to 1 +- Numbers should not contain commas - **c2<=44,554,350** will not work, but **c2<=44554350** will +- Some words in the data can be used, but must be single or double quoted ( e.g., **c3=='exon'** ) + + + + From efa2f58d3feb0ac272493df85d876750e7fd87d9 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 1 Dec 2023 11:53:08 +0100 Subject: [PATCH 2/4] also remove duplicates in sections --- lib/galaxy/tool_util/toolbox/views/static.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/toolbox/views/static.py b/lib/galaxy/tool_util/toolbox/views/static.py index 2eba8eef3f4b..7972cc84e835 100644 --- a/lib/galaxy/tool_util/toolbox/views/static.py +++ b/lib/galaxy/tool_util/toolbox/views/static.py @@ -105,7 +105,7 @@ def definition_with_items_to_panel(definition, allow_sections: bool = True, item f"Failed to find matching section for (id, name) = ({section_def.id}, {section_def.name})" ) continue - section = closest_section.copy() + section = closest_section.copy(merge_tools=True) if section_def.id is not None: section.id = section_def.id if section_def.name is not None: From db9cd1cd49effb4b31d36afe15fb2c6148dafc78 Mon Sep 17 00:00:00 2001 From: Alexander OSTROVSKY Date: Wed, 10 Jan 2024 15:56:31 -0800 Subject: [PATCH 3/4] add binary datatypes for intermediate output of fastk tools --- lib/galaxy/config/sample/datatypes_conf.xml.sample | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index f97bf1f54c73..e2d80eae608d 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -210,6 +210,9 @@ + + + From 01502917f761f1e6a3e9ab9c0fd01b5ca073b6ee Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 12 Jan 2024 12:24:51 +0100 Subject: [PATCH 4/4] Fix ``to_cwl`` for nested collections Fixes ``` TypeError Object of type DatasetCollection is not JSON serializable ``` seen when running https://github.com/galaxyproject/iwc/pull/315 --- lib/galaxy/workflow/modules.py | 11 ++++++----- test/unit/workflows/test_modules.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 805a8ba3f2f6..a187d623f1c1 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -123,6 +123,8 @@ class ConditionalStepWhen(BooleanToolParameter): def to_cwl(value, hda_references, step): element_identifier = None + if isinstance(value, model.HistoryDatasetCollectionAssociation): + value = value.collection if isinstance(value, model.DatasetCollectionElement) and value.hda: element_identifier = value.element_identifier value = value.hda @@ -152,14 +154,13 @@ def to_cwl(value, hda_references, step): properties, value.dataset.created_from_basename or element_identifier or value.name ) return properties - elif hasattr(value, "collection"): - collection = value.collection - if collection.collection_type == "list": - return [to_cwl(dce, hda_references=hda_references, step=step) for dce in collection.dataset_elements] + elif isinstance(value, model.DatasetCollection): + if value.collection_type == "list": + return [to_cwl(dce, hda_references=hda_references, step=step) for dce in value.dataset_elements] else: # Could be record or nested lists rval = {} - for element in collection.elements: + for element in value.elements: rval[element.element_identifier] = to_cwl( element.element_object, hda_references=hda_references, step=step ) diff --git a/test/unit/workflows/test_modules.py b/test/unit/workflows/test_modules.py index 51dc049b7562..4f88e33bfd14 100644 --- a/test/unit/workflows/test_modules.py +++ b/test/unit/workflows/test_modules.py @@ -261,6 +261,19 @@ def test_to_cwl(): assert hda_references == hdas +def test_to_cwl_nested_collection(): + hda = model.HistoryDatasetAssociation(create_dataset=True, flush=False) + hda.dataset.state = model.Dataset.states.OK + dc_inner = model.DatasetCollection(collection_type="list") + model.DatasetCollectionElement(collection=dc_inner, element_identifier="inner", element=hda) + dc_outer = model.DatasetCollection(collection_type="list:list") + model.DatasetCollectionElement(collection=dc_outer, element_identifier="outer", element=dc_inner) + hdca = model.HistoryDatasetCollectionAssociation(name="the collection", collection=dc_outer) + result = modules.to_cwl(hdca, [], model.WorkflowStep()) + assert result["outer"][0]["class"] == "File" + assert result["outer"][0]["basename"] == "inner" + + class MapOverTestCase(NamedTuple): data_input: str step_input_def: Union[str, List[str]]