Merge branch 'galaxyproject:dev' into FastAPI_library_contents

doc/source/admin/galaxy_options.rst

@@ -5115,6 +5115,26 @@
 :Type: str
 
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``workflow_scheduling_separate_materialization_iteration``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Workflows launched with URI/URL inputs that are not marked as
+    'deferred' are "materialized" (or undeferred) by the workflow
+    scheduler. This might be a lengthy process. Setting this to 'True'
+    will place the invocation back in the queue after materialization
+    before scheduling the workflow so it is less  likely to starve
+    other workflow scheduling. Ideally, Galaxy would allow more fine
+    grain control of handlers but until then, this provides a way to
+    tip the balance between "doing more work" and "being more fair".
+    The default here is pretty arbitrary - it has been to False to
+    optimize Galaxy for automated, single user applications where
+    "fairness" is mostly irrelevant.
+:Default: ``false``
+:Type: bool
+
+
 ~~~~~~~~~~~~~~~~~~~~~~~~
 ``cache_user_job_count``
 ~~~~~~~~~~~~~~~~~~~~~~~~
@@ -5602,3 +5622,42 @@
     This requires the help_forum_api_url to be set.
 :Default: ``false``
 :Type: bool
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~
+``file_source_temp_dir``
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Directory to store temporary files for file sources. This defaults
+    to new_file_path if not set.
+:Default: ``None``
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``file_source_webdav_use_temp_files``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Default value for use_temp_files for webdav plugins that don't
+    explicitly declare this.
+:Default: ``true``
+:Type: bool
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``file_source_listings_expiry_time``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Number of seconds before file source content listings are
+    refreshed. Shorter times will result in more queries while
+    browsing a file sources. Longer times will result in fewer
+    requests to file sources but outdated contents might be displayed
+    to the user. Currently only affects s3fs file sources.
+:Default: ``60``
+:Type: int
+
+
+

lib/galaxy/config/__init__.py

@@ -704,6 +704,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
     drmaa_external_runjob_script: str
     email_from: Optional[str]
     enable_tool_shed_check: bool
+    file_source_temp_dir: str
     galaxy_data_manager_data_path: str
     galaxy_infrastructure_url: str
     hours_between_check: int
@@ -1236,6 +1237,9 @@ def _load_theme(path: str, theme_dict: dict):
         else:
             _load_theme(self.themes_config_file, self.themes)
 
+        if self.file_source_temp_dir:
+            self.file_source_temp_dir = os.path.abspath(self.file_source_temp_dir)
+
     def _process_celery_config(self):
         if self.celery_conf and self.celery_conf.get("result_backend") is None:
             # If the result_backend is not set, use a SQLite database in the data directory
@@ -1348,6 +1352,7 @@ def check(self):
             self.template_cache_path,
             self.tool_data_path,
             self.user_library_import_dir,
+            self.file_source_temp_dir,
         ]
         for path in paths_to_check:
             self._ensure_directory(path)

lib/galaxy/config/sample/galaxy.yml.sample

@@ -2748,6 +2748,19 @@ galaxy:
   # <config_dir>.
   #workflow_schedulers_config_file: workflow_schedulers_conf.xml
 
+  # Workflows launched with URI/URL inputs that are not marked as
+  # 'deferred' are "materialized" (or undeferred) by the workflow
+  # scheduler. This might be a lengthy process. Setting this to 'True'
+  # will place the invocation back in the queue after materialization
+  # before scheduling the workflow so it is less  likely to starve other
+  # workflow scheduling. Ideally, Galaxy would allow more fine grain
+  # control of handlers but until then, this provides a way to tip the
+  # balance between "doing more work" and "being more fair". The default
+  # here is pretty arbitrary - it has been to False to optimize Galaxy
+  # for automated, single user applications where "fairness" is mostly
+  # irrelevant.
+  #workflow_scheduling_separate_materialization_iteration: false
+
   # If using job concurrency limits (configured in job_config_file),
   # several extra database queries must be performed to determine the
   # number of jobs a user has dispatched to a given destination.  By
@@ -2978,3 +2991,19 @@ galaxy:
   # Enable the integration of the Galaxy Help Forum in the tool panel.
   # This requires the help_forum_api_url to be set.
   #enable_help_forum_tool_panel_integration: false
+
+  # Directory to store temporary files for file sources. This defaults
+  # to new_file_path if not set.
+  #file_source_temp_dir: null
+
+  # Default value for use_temp_files for webdav plugins that don't
+  # explicitly declare this.
+  #file_source_webdav_use_temp_files: true
+
+  # Number of seconds before file source content listings are refreshed.
+  # Shorter times will result in more queries while browsing a file
+  # sources. Longer times will result in fewer requests to file sources
+  # but outdated contents might be displayed to the user. Currently only
+  # affects s3fs file sources.
+  #file_source_listings_expiry_time: 60
+

lib/galaxy/config/sample/tool_conf.xml.sample

@@ -32,6 +32,7 @@
     <tool file="${model_tools_path}/zip_collection.xml" />
     <tool file="${model_tools_path}/filter_failed_collection.xml" />
     <tool file="${model_tools_path}/filter_empty_collection.xml" />
+    <tool file="${model_tools_path}/filter_null.xml" />
     <tool file="${model_tools_path}/flatten_collection.xml" />
     <tool file="${model_tools_path}/merge_collection.xml" />
     <tool file="${model_tools_path}/relabel_from_file.xml" />

lib/galaxy/config/schemas/config_schema.yml

@@ -4104,3 +4104,23 @@ mapping:
         per_host: true
         desc: |
           Enable the integration of the Galaxy Help Forum in the tool panel. This requires the help_forum_api_url to be set.
+
+      file_source_temp_dir:
+        type: str
+        required: false
+        desc: |
+          Directory to store temporary files for file sources. This defaults to new_file_path if not set.
+      
+      file_source_webdav_use_temp_files:
+        type: bool
+        default: true
+        desc: |
+          Default value for use_temp_files for webdav plugins that don't explicitly declare this.
+
+      file_source_listings_expiry_time:
+        type: int
+        default: 60
+        desc: |
+          Number of seconds before file source content listings are refreshed. Shorter times will result in more
+          queries while browsing a file sources. Longer times will result in fewer requests to file sources but
+          outdated contents might be displayed to the user. Currently only affects s3fs file sources.

lib/galaxy/dependencies/dev-requirements.txt

@@ -1,5 +1,7 @@
-aiohappyeyeballs==2.4.0 ; python_version >= "3.8" and python_version < "3.13"
-aiohttp==3.10.5 ; python_version >= "3.8" and python_version < "3.13"
+--extra-index-url https://wheels.galaxyproject.org/simple
+
+aiohappyeyeballs==2.4.3 ; python_version >= "3.8" and python_version < "3.13"
+aiohttp==3.10.9 ; python_version >= "3.8" and python_version < "3.13"
 aiosignal==1.3.1 ; python_version >= "3.8" and python_version < "3.13"
 alabaster==0.7.13 ; python_version >= "3.8" and python_version < "3.13"
 anyio==4.5.0 ; python_version >= "3.8" and python_version < "3.13"
@@ -37,13 +39,13 @@ dogpile-cache==1.3.3 ; python_version >= "3.9" and python_version < "3.13"
 exceptiongroup==1.2.2 ; python_version >= "3.8" and python_version < "3.11"
 filelock==3.16.1 ; python_version >= "3.8" and python_version < "3.13"
 fluent-logger==0.11.1 ; python_version >= "3.8" and python_version < "3.13"
-fonttools==4.53.1 ; python_version >= "3.8" and python_version < "3.13"
+fonttools==4.54.1 ; python_version >= "3.8" and python_version < "3.13"
 frozenlist==1.4.1 ; python_version >= "3.8" and python_version < "3.13"
 galaxy-release-util==0.1.11 ; python_version >= "3.8" and python_version < "3.13"
 graylint==1.1.1 ; python_version >= "3.8" and python_version < "3.13"
 greenlet==3.0.3 ; python_version >= "3.8" and python_version < "3.13"
 h11==0.14.0 ; python_version >= "3.8" and python_version < "3.13"
-httpcore==1.0.5 ; python_version >= "3.8" and python_version < "3.13"
+httpcore==1.0.6 ; python_version >= "3.8" and python_version < "3.13"
 httpx==0.27.2 ; python_version >= "3.8" and python_version < "3.13"
 idna==3.10 ; python_version >= "3.8" and python_version < "3.13"
 imagesize==1.4.1 ; python_version >= "3.8" and python_version < "3.13"
@@ -54,7 +56,7 @@ isodate==0.6.1 ; python_version >= "3.8" and python_version < "3.13"
 isort==5.13.2 ; python_version >= "3.8" and python_version < "3.13"
 jaraco-classes==3.4.0 ; python_version >= "3.8" and python_version < "3.13"
 jaraco-context==6.0.1 ; python_version >= "3.8" and python_version < "3.13"
-jaraco-functools==4.0.2 ; python_version >= "3.8" and python_version < "3.13"
+jaraco-functools==4.1.0 ; python_version >= "3.8" and python_version < "3.13"
 jeepney==0.8.0 ; python_version >= "3.8" and python_version < "3.13" and sys_platform == "linux"
 jinja2==3.1.4 ; python_version >= "3.8" and python_version < "3.13"
 jmespath==1.0.1 ; python_version >= "3.8" and python_version < "3.13"
@@ -96,15 +98,15 @@ pluggy==1.5.0 ; python_version >= "3.8" and python_version < "3.13"
 port-for==0.7.3 ; python_version >= "3.8" and python_version < "3.13"
 prettytable==3.11.0 ; python_version >= "3.8" and python_version < "3.13"
 psutil==6.0.0 ; python_version >= "3.8" and python_version < "3.13" and sys_platform != "cygwin"
-psycopg==3.2.2 ; python_version >= "3.8" and python_version < "3.13"
+psycopg==3.2.3 ; python_version >= "3.8" and python_version < "3.13"
 pycparser==2.22 ; python_version >= "3.8" and python_version < "3.13"
 pyee==12.0.0 ; python_version >= "3.8" and python_version < "3.13"
 pygithub==2.4.0 ; python_version >= "3.8" and python_version < "3.13"
 pygments==2.18.0 ; python_version >= "3.8" and python_version < "3.13"
 pyjwt[crypto]==2.9.0 ; python_version >= "3.8" and python_version < "3.13"
 pynacl==1.5.0 ; python_version >= "3.8" and python_version < "3.13"
 pyparsing==3.1.4 ; python_version >= "3.8" and python_version < "3.13"
-pyproject-hooks==1.1.0 ; python_version >= "3.8" and python_version < "3.13"
+pyproject-hooks==1.2.0 ; python_version >= "3.8" and python_version < "3.13"
 pysocks==1.7.1 ; python_version >= "3.8" and python_version < "3.13"
 pytest-asyncio==0.23.8 ; python_version >= "3.8" and python_version < "3.13"
 pytest-base-url==2.1.0 ; python_version >= "3.8" and python_version < "3.13"
@@ -131,11 +133,11 @@ requests-toolbelt==1.0.0 ; python_version >= "3.8" and python_version < "3.13"
 requests==2.32.3 ; python_version >= "3.8" and python_version < "3.13"
 responses==0.25.3 ; python_version >= "3.8" and python_version < "3.13"
 rfc3986==2.0.0 ; python_version >= "3.8" and python_version < "3.13"
-rich==13.8.1 ; python_version >= "3.8" and python_version < "3.13"
+rich==13.9.2 ; python_version >= "3.8" and python_version < "3.13"
 rpds-py==0.20.0 ; python_version >= "3.9" and python_version < "3.13"
 ruamel-yaml-clib==0.2.8 ; platform_python_implementation == "CPython" and python_version < "3.13" and python_version >= "3.8"
 ruamel-yaml==0.18.6 ; python_version >= "3.8" and python_version < "3.13"
-rucio-clients==35.3.0 ; python_version >= "3.9" and python_version < "3.13"
+rucio-clients==35.4.0 ; python_version >= "3.9" and python_version < "3.13"
 s3transfer==0.10.2 ; python_version >= "3.8" and python_version < "3.13"
 schema-salad==8.7.20240905150001 ; python_version >= "3.8" and python_version < "3.13"
 scipy==1.10.1 ; python_version >= "3.8" and python_version < "3.9"
@@ -165,14 +167,14 @@ testfixtures==8.3.0 ; python_version >= "3.8" and python_version < "3.13"
 text-unidecode==1.3 ; python_version >= "3.8" and python_version < "3.13"
 tinydb==4.8.0 ; python_version >= "3.8" and python_version < "3.13"
 toml==0.10.2 ; python_version >= "3.8" and python_version < "3.13"
-tomli==2.0.1 ; python_version >= "3.8" and python_full_version <= "3.11.0a6"
+tomli==2.0.2 ; python_version >= "3.8" and python_full_version <= "3.11.0a6"
 trio-websocket==0.11.1 ; python_version >= "3.8" and python_version < "3.13"
 trio==0.26.2 ; python_version >= "3.8" and python_version < "3.13"
 tuspy==1.0.3 ; python_version >= "3.8" and python_version < "3.13"
 twill==3.2.5 ; python_version >= "3.8" and python_version < "3.13"
 twine==5.1.1 ; python_version >= "3.8" and python_version < "3.13"
 typing-extensions==4.12.2 ; python_version >= "3.8" and python_version < "3.13"
-tzdata==2024.1 ; python_version >= "3.8" and python_version < "3.13" and sys_platform == "win32"
+tzdata==2024.2 ; python_version >= "3.8" and python_version < "3.13" and sys_platform == "win32"
 urllib3==1.26.20 ; python_version >= "3.8" and python_version < "3.13"
 urllib3[socks]==1.26.20 ; python_version >= "3.8" and python_version < "3.13"
 watchdog==4.0.2 ; python_version >= "3.8" and python_version < "3.13"
@@ -181,5 +183,5 @@ websocket-client==1.8.0 ; python_version >= "3.8" and python_version < "3.13"
 werkzeug==3.0.4 ; python_version >= "3.8" and python_version < "3.13"
 wrapt==1.16.0 ; python_version >= "3.8" and python_version < "3.13"
 wsproto==1.2.0 ; python_version >= "3.8" and python_version < "3.13"
-yarl==1.11.1 ; python_version >= "3.8" and python_version < "3.13"
+yarl==1.13.1 ; python_version >= "3.8" and python_version < "3.13"
 zipp==3.20.2 ; python_version >= "3.8" and python_version < "3.13"

lib/galaxy/dependencies/pinned-lint-requirements.txt

@@ -4,4 +4,4 @@ flake8-bugbear==24.8.19
 mccabe==0.7.0
 pycodestyle==2.12.1
 pyflakes==3.2.0
-ruff==0.6.6
+ruff==0.6.9