Merge branch 'release_24.0' into dev

lib/galaxy/managers/workflows.py

@@ -89,12 +89,14 @@
     visit_input_values,
 )
 from galaxy.tools.parameters.basic import (
-    ConnectedValue,
     DataCollectionToolParameter,
     DataToolParameter,
+)
+from galaxy.tools.parameters.workflow_utils import (
+    ConnectedValue,
     RuntimeValue,
+    workflow_building_modes,
 )
-from galaxy.tools.parameters.workflow_building_modes import workflow_building_modes
 from galaxy.util.hash_util import md5_hash_str
 from galaxy.util.json import (
     safe_dumps,

lib/galaxy/tool_util/verify/__init__.py

@@ -31,11 +31,12 @@
 try:
     from PIL import Image
 except ImportError:
-    pass
+    Image = None  # type: ignore[assignment]
 try:
     import tifffile
 except ImportError:
-    pass
+    tifffile = None  # type: ignore[assignment]
+
 
 from galaxy.tool_util.parser.util import (
     DEFAULT_DELTA,
@@ -193,7 +194,12 @@ def get_filename(filename: str) -> str:
             elif compare == "contains":
                 files_contains(local_name, temp_name, attributes=attributes)
             elif compare == "image_diff":
-                files_image_diff(local_name, temp_name, attributes=attributes)
+                if Image and tifffile:
+                    files_image_diff(local_name, temp_name, attributes=attributes)
+                else:
+                    raise Exception(
+                        "pillow and tifffile are not installed, but required to compare files using the 'image_diff' method"
+                    )
             else:
                 raise Exception(f"Unimplemented Compare type: {compare}")
         except AssertionError as err:

lib/galaxy/tools/__init__.py

@@ -142,7 +142,7 @@
 )
 from galaxy.tools.parameters.input_translation import ToolInputTranslator
 from galaxy.tools.parameters.meta import expand_meta_parameters
-from galaxy.tools.parameters.workflow_building_modes import workflow_building_modes
+from galaxy.tools.parameters.workflow_utils import workflow_building_modes
 from galaxy.tools.parameters.wrapped_json import json_wrap
 from galaxy.tools.test import parse_tests
 from galaxy.util import (

lib/galaxy/tools/actions/__init__.py

@@ -33,9 +33,9 @@
 from galaxy.tools.parameters.basic import (
     DataCollectionToolParameter,
     DataToolParameter,
-    RuntimeValue,
     SelectToolParameter,
 )
+from galaxy.tools.parameters.workflow_utils import RuntimeValue
 from galaxy.tools.parameters.wrapped import (
     LegacyUnprefixedDict,
     WrappedParameters,

lib/galaxy/tools/execute.py

@@ -34,7 +34,7 @@
     on_text_for_names,
     ToolExecutionCache,
 )
-from galaxy.tools.parameters.basic import is_runtime_value
+from galaxy.tools.parameters.workflow_utils import is_runtime_value
 
 if typing.TYPE_CHECKING:
     from galaxy.tools import Tool

lib/galaxy/tools/parameters/__init__.py

@@ -16,9 +16,7 @@
 from .basic import (
     DataCollectionToolParameter,
     DataToolParameter,
-    is_runtime_value,
     ParameterValueError,
-    runtime_to_json,
     SelectToolParameter,
     ToolParameter,
 )
@@ -29,6 +27,11 @@
     Section,
     UploadDataset,
 )
+from .workflow_utils import (
+    is_runtime_value,
+    runtime_to_json,
+)
+from .wrapped import flat_to_nested_state
 
 REPLACE_ON_TRUTHY = object()
 
@@ -515,6 +518,8 @@ def populate_state(
 def _populate_state_legacy(
     request_context, inputs, incoming, state, errors, prefix="", context=None, check=True, simple_errors=True
 ):
+    if context is None:
+        context = flat_to_nested_state(incoming)
     context = ExpressionContext(state, context)
     for input in inputs.values():
         state[input.name] = input.get_initial_value(request_context, context)
@@ -600,8 +605,8 @@ def _populate_state_legacy(
                 for upload_item in input.inputs.values():
                     new_state[upload_item.name] = upload_item.get_initial_value(request_context, context)
                 group_state.append(new_state)
-            for rep_state in group_state:
-                rep_index = rep_state["__index__"]
+            for rep_index, rep_state in enumerate(group_state):
+                rep_index = rep_state.get("__index__", rep_index)
                 rep_prefix = "%s_%d|" % (key, rep_index)
                 _populate_state_legacy(
                     request_context,

lib/galaxy/tools/parameters/basic.py

@@ -42,7 +42,7 @@
 from galaxy.model.dataset_collections import builder
 from galaxy.schema.fetch_data import FilesPayload
 from galaxy.tool_util.parser import get_input_source as ensure_input_source
-from galaxy.tools.parameters.workflow_building_modes import workflow_building_modes
+from galaxy.tools.parameters.workflow_utils import workflow_building_modes
 from galaxy.util import (
     sanitize_param,
     string_as_bool,
@@ -61,6 +61,12 @@
 )
 from .dataset_matcher import get_dataset_matcher_factory
 from .sanitize import ToolParameterSanitizer
+from .workflow_utils import (
+    is_runtime_value,
+    runtime_to_json,
+    runtime_to_object,
+    RuntimeValue,
+)
 
 if TYPE_CHECKING:
     from sqlalchemy.orm import Session
@@ -92,12 +98,6 @@ def contains_workflow_parameter(value, search=False):
     return False
 
 
-def is_runtime_value(value):
-    return isinstance(value, RuntimeValue) or (
-        isinstance(value, MutableMapping) and value.get("__class__") in ["RuntimeValue", "ConnectedValue"]
-    )
-
-
 def is_runtime_context(trans, other_values):
     if trans.workflow_building_mode:
         return True
@@ -2780,36 +2780,6 @@ def write_elements_to_collection(has_elements, collection_builder):
 )
 
 
-def runtime_to_json(runtime_value):
-    if isinstance(runtime_value, ConnectedValue) or (
-        isinstance(runtime_value, MutableMapping) and runtime_value["__class__"] == "ConnectedValue"
-    ):
-        return {"__class__": "ConnectedValue"}
-    else:
-        return {"__class__": "RuntimeValue"}
-
-
-def runtime_to_object(runtime_value):
-    if isinstance(runtime_value, ConnectedValue) or (
-        isinstance(runtime_value, MutableMapping) and runtime_value["__class__"] == "ConnectedValue"
-    ):
-        return ConnectedValue()
-    else:
-        return RuntimeValue()
-
-
-class RuntimeValue:
-    """
-    Wrapper to note a value that is not yet set, but will be required at runtime.
-    """
-
-
-class ConnectedValue(RuntimeValue):
-    """
-    Wrapper to note a value that is not yet set, but will be inferred from a connection.
-    """
-
-
 def history_item_dict_to_python(value, app, name):
     if isinstance(value, MutableMapping) and "src" in value:
         if value["src"] not in ("hda", "dce", "ldda", "hdca"):

lib/galaxy/tools/parameters/dynamic_options.py

@@ -28,7 +28,10 @@
     User,
 )
 from galaxy.tools.expressions import do_eval
-from galaxy.tools.parameters.workflow_building_modes import workflow_building_modes
+from galaxy.tools.parameters.workflow_utils import (
+    is_runtime_value,
+    workflow_building_modes,
+)
 from galaxy.util import (
     Element,
     string_as_bool,
@@ -737,16 +740,20 @@ def get_fields(self, trans, other_values):
                 if not hasattr(dataset, "get_file_name"):
                     continue
                 # Ensure parsing dynamic options does not consume more than a megabyte worth memory.
-                path = dataset.get_file_name()
-                if os.path.getsize(path) < 1048576:
-                    with open(path) as fh:
-                        options += self.parse_file_fields(fh)
-                else:
-                    # Pass just the first megabyte to parse_file_fields.
-                    log.warning("Attempting to load options from large file, reading just first megabyte")
-                    with open(path) as fh:
-                        contents = fh.read(1048576)
-                    options += self.parse_file_fields(StringIO(contents))
+                try:
+                    path = dataset.get_file_name()
+                    if os.path.getsize(path) < 1048576:
+                        with open(path) as fh:
+                            options += self.parse_file_fields(fh)
+                    else:
+                        # Pass just the first megabyte to parse_file_fields.
+                        log.warning("Attempting to load options from large file, reading just first megabyte")
+                        with open(path) as fh:
+                            contents = fh.read(1048576)
+                        options += self.parse_file_fields(StringIO(contents))
+                except Exception as e:
+                    log.warning("Could not read contents from %s: %s", dataset, str(e))
+                    continue
         elif self.tool_data_table:
             options = self.tool_data_table.get_fields()
             if trans and trans.user and trans.workflow_building_mode != workflow_building_modes.ENABLED:
@@ -964,6 +971,8 @@ def _get_ref_data(other_values, ref_name):
             list,
         ),
     ):
+        if is_runtime_value(ref):
+            return []
         raise ValueError
     if isinstance(ref, DatasetCollectionElement) and ref.hda:
         ref = ref.hda

lib/galaxy/tools/parameters/grouping.py

@@ -618,8 +618,8 @@ def get_filenames(context):
         writable_files = d_type.writable_files
         writable_files_offset = 0
         groups_incoming = [None for _ in range(file_count)]
-        for group_incoming in context.get(self.name, []):
-            i = int(group_incoming["__index__"])
+        for i, group_incoming in enumerate(context.get(self.name, [])):
+            i = int(group_incoming.get("__index__", i))
             groups_incoming[i] = group_incoming
         if d_type.composite_type is not None or force_composite:
             # handle uploading of composite datatypes

lib/galaxy/tools/parameters/meta.py

@@ -21,6 +21,7 @@
 )
 from galaxy.util import permutations
 from . import visit_input_values
+from .wrapped import process_key
 
 log = logging.getLogger(__name__)
 
@@ -153,31 +154,6 @@ def is_batch(value):
     return WorkflowParameterExpansion(param_combinations, params_keys, input_combinations)
 
 
-def process_key(incoming_key, incoming_value, d):
-    key_parts = incoming_key.split("|")
-    if len(key_parts) == 1:
-        # Regular parameter
-        if incoming_key in d and not incoming_value:
-            # In case we get an empty repeat after we already filled in a repeat element
-            return
-        d[incoming_key] = incoming_value
-    elif key_parts[0].rsplit("_", 1)[-1].isdigit():
-        # Repeat
-        input_name, index = key_parts[0].rsplit("_", 1)
-        index = int(index)
-        d.setdefault(input_name, [])
-        newlist = [{} for _ in range(index + 1)]
-        d[input_name].extend(newlist[len(d[input_name]) :])
-        subdict = d[input_name][index]
-        process_key("|".join(key_parts[1:]), incoming_value=incoming_value, d=subdict)
-    else:
-        # Section / Conditional
-        input_name = key_parts[0]
-        subdict = {}
-        d[input_name] = subdict
-        process_key("|".join(key_parts[1:]), incoming_value=incoming_value, d=subdict)
-
-
 ExpandedT = Tuple[List[Dict[str, Any]], Optional[matching.MatchingCollections]]
 
 

lib/galaxy/tools/parameters/workflow_utils.py

@@ -0,0 +1,43 @@
+from collections.abc import MutableMapping
+
+
+class workflow_building_modes:
+    DISABLED = False
+    ENABLED = True
+    USE_HISTORY = 1
+
+
+def runtime_to_json(runtime_value):
+    if isinstance(runtime_value, ConnectedValue) or (
+        isinstance(runtime_value, MutableMapping) and runtime_value["__class__"] == "ConnectedValue"
+    ):
+        return {"__class__": "ConnectedValue"}
+    else:
+        return {"__class__": "RuntimeValue"}
+
+
+def runtime_to_object(runtime_value):
+    if isinstance(runtime_value, ConnectedValue) or (
+        isinstance(runtime_value, MutableMapping) and runtime_value["__class__"] == "ConnectedValue"
+    ):
+        return ConnectedValue()
+    else:
+        return RuntimeValue()
+
+
+class RuntimeValue:
+    """
+    Wrapper to note a value that is not yet set, but will be required at runtime.
+    """
+
+
+class ConnectedValue(RuntimeValue):
+    """
+    Wrapper to note a value that is not yet set, but will be inferred from a connection.
+    """
+
+
+def is_runtime_value(value):
+    return isinstance(value, RuntimeValue) or (
+        isinstance(value, MutableMapping) and value.get("__class__") in ["RuntimeValue", "ConnectedValue"]
+    )

lib/galaxy/tools/parameters/wrapped.py

@@ -1,5 +1,9 @@
 from collections import UserDict
-from typing import Dict
+from typing import (
+    Any,
+    Dict,
+    List,
+)
 
 from galaxy.tools.parameters.basic import (
     DataCollectionToolParameter,
@@ -158,4 +162,36 @@ def make_list_copy(from_list):
     return new_list
 
 
-__all__ = ("LegacyUnprefixedDict", "WrappedParameters", "make_dict_copy")
+def process_key(incoming_key: str, incoming_value: Any, d: Dict[str, Any]):
+    key_parts = incoming_key.split("|")
+    if len(key_parts) == 1:
+        # Regular parameter
+        if incoming_key in d and not incoming_value:
+            # In case we get an empty repeat after we already filled in a repeat element
+            return
+        d[incoming_key] = incoming_value
+    elif key_parts[0].rsplit("_", 1)[-1].isdigit():
+        # Repeat
+        input_name, _index = key_parts[0].rsplit("_", 1)
+        index = int(_index)
+        d.setdefault(input_name, [])
+        newlist: List[Dict[Any, Any]] = [{} for _ in range(index + 1)]
+        d[input_name].extend(newlist[len(d[input_name]) :])
+        subdict = d[input_name][index]
+        process_key("|".join(key_parts[1:]), incoming_value=incoming_value, d=subdict)
+    else:
+        # Section / Conditional
+        input_name = key_parts[0]
+        subdict = d.get(input_name, {})
+        d[input_name] = subdict
+        process_key("|".join(key_parts[1:]), incoming_value=incoming_value, d=subdict)
+
+
+def flat_to_nested_state(incoming: Dict[str, Any]):
+    nested_state: Dict[str, Any] = {}
+    for key, value in incoming.items():
+        process_key(key, value, nested_state)
+    return nested_state
+
+
+__all__ = ("LegacyUnprefixedDict", "WrappedParameters", "make_dict_copy", "process_key", "flat_to_nested_state")

lib/galaxy/tools/recommendations.py

@@ -10,7 +10,7 @@
 import yaml
 
 from galaxy.tools.parameters import populate_state
-from galaxy.tools.parameters.workflow_building_modes import workflow_building_modes
+from galaxy.tools.parameters.workflow_utils import workflow_building_modes
 from galaxy.util import DEFAULT_SOCKET_TIMEOUT
 from galaxy.workflow.modules import module_factory