diff --git a/planemo/engine/factory.py b/planemo/engine/factory.py index 7d57b7d3b..3d1ac1993 100644 --- a/planemo/engine/factory.py +++ b/planemo/engine/factory.py @@ -10,6 +10,7 @@ ExternalGalaxyEngine, LocalManagedGalaxyEngine, LocalManagedGalaxyEngineWithSingularityDB, + UvxManagedGalaxyEngine, ) from .toil import ToilEngine @@ -19,7 +20,7 @@ def is_galaxy_engine(**kwds): """Return True iff the engine configured is :class:`GalaxyEngine`.""" engine_type_str = kwds.get("engine", "galaxy") - return engine_type_str in ["galaxy", "docker_galaxy", "external_galaxy"] + return engine_type_str in ["galaxy", "docker_galaxy", "external_galaxy", "uvx_galaxy"] def build_engine(ctx, **kwds): @@ -34,6 +35,8 @@ def build_engine(ctx, **kwds): engine_type = DockerizedManagedGalaxyEngine elif engine_type_str == "external_galaxy": engine_type = ExternalGalaxyEngine + elif engine_type_str == "uvx_galaxy": + engine_type = UvxManagedGalaxyEngine elif engine_type_str == "cwltool": engine_type = CwlToolEngine elif engine_type_str == "toil": diff --git a/planemo/engine/galaxy.py b/planemo/engine/galaxy.py index c56862dab..c43f4ea0d 100644 --- a/planemo/engine/galaxy.py +++ b/planemo/engine/galaxy.py @@ -145,7 +145,10 @@ def ensure_runnables_served(self, runnables): with serve_daemon(self._ctx, runnables, **self._serve_kwds()) as config: if "install_args_list" in self._serve_kwds(): self.shed_install(config) - yield config + try: + yield config + finally: + config.kill() def shed_install(self, config): kwds = self._serve_kwds() @@ -191,6 +194,18 @@ def _serve_kwds(self): return serve_kwds +class UvxManagedGalaxyEngine(LocalManagedGalaxyEngine): + """An :class:`Engine` implementation backed by Galaxy running via uvx. + + More information on Galaxy can be found at http://galaxyproject.org/. + """ + + def _serve_kwds(self): + serve_kwds = self._kwds.copy() + serve_kwds["uvx_galaxy"] = True + return serve_kwds + + class ExternalGalaxyEngine(GalaxyEngine): """An :class:`Engine` implementation backed by an external Galaxy instance.""" diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py index 694f884b0..4a6072b01 100644 --- a/planemo/galaxy/config.py +++ b/planemo/galaxy/config.py @@ -9,6 +9,7 @@ import random import shlex import shutil +import subprocess import threading import time from string import Template @@ -158,6 +159,8 @@ def galaxy_config(ctx, runnables, **kwds): c = docker_galaxy_config elif kwds.get("external", False): c = external_galaxy_config + elif kwds.get("uvx_galaxy", False): + c = uvx_galaxy_config log_thread = None e = threading.Event() try: @@ -1421,7 +1424,301 @@ def _ensure_directory(path): os.makedirs(path) +class UvxGalaxyConfig(BaseManagedGalaxyConfig): + """A uvx-managed implementation of :class:`GalaxyConfig`.""" + + def __init__( + self, + ctx, + config_directory, + env, + test_data_dir, + port, + server_name, + master_api_key, + runnables, + kwds, + ): + super().__init__( + ctx, + config_directory, + env, + test_data_dir, + port, + server_name, + master_api_key, + runnables, + kwds, + ) + # Use config directory as placeholder for galaxy_root since uvx manages Galaxy + self.galaxy_root = config_directory + + @property + def host(self): + """Host for uvx Galaxy instance.""" + return self._kwds.get("host", "127.0.0.1") + + @property + def galaxy_config_file(self): + """Path to galaxy configuration file.""" + return self.env.get("GALAXY_CONFIG_FILE", os.path.join(self.config_directory, "galaxy.yml")) + + def kill(self): + """Kill uvx Galaxy process.""" + if self._ctx.verbose: + shell(["ps", "ax"]) + exists = os.path.exists(self.pid_file) + print(f"Killing pid file [{self.pid_file}]") + print(f"pid_file exists? [{exists}]") + if exists: + with open(self.pid_file) as f: + print(f"pid_file contents are [{f.read()}]") + + # Kill process using existing utility + kill_pid_file(self.pid_file) + + def startup_command(self, ctx, **kwds): + """Return a shell command used to startup this uvx Galaxy instance.""" + daemon = kwds.get("daemon", False) + uvx_cmd = self._build_uvx_command(**kwds) + + if daemon: + # Use shell background execution for daemon mode - return as single string for shell execution + return f"nohup {shell_join(uvx_cmd)} > {self.log_file} 2>&1 & echo $! > {self.pid_file}" + else: + # Direct foreground execution + return shell_join(uvx_cmd) + + def _build_uvx_command(self, **kwds): + """Build uvx galaxy command with appropriate flags.""" + with NamedTemporaryFile("wb", prefix="planemo_galaxy_extra_deps", suffix=".txt", delete=False) as extra_deps: + p = subprocess.run( + [ + "uvx", + "--from", + "galaxy", + "galaxy-dependencies", + "--freeze", + "--config_file", + self.galaxy_config_file, + ], + stdout=extra_deps, + ) + p.check_returncode() + + cmd = ["uvx", "--with-requirements", extra_deps.name, "galaxy"] + + # Only pass config file - host and port are configured in galaxy.yml + cmd.extend(["-c", self.galaxy_config_file]) + + return cmd + + @property + def log_file(self): + """Log file used when planemo serves this uvx Galaxy instance.""" + file_name = f"{self.server_name}.log" + return os.path.join(self.config_directory, file_name) + + @property + def pid_file(self): + """PID file for uvx Galaxy process.""" + pid_file_name = f"{self.server_name}.pid" + return os.path.join(self.config_directory, pid_file_name) + + @property + def log_contents(self): + """Return contents of log file.""" + if not os.path.exists(self.log_file): + return "" + with open(self.log_file) as f: + return f.read() + + def cleanup(self): + """Clean up uvx Galaxy configuration.""" + shutil.rmtree(self.config_directory, CLEANUP_IGNORE_ERRORS) + + @property + def default_use_path_paste(self): + """Default path paste setting for uvx Galaxy.""" + return self.user_is_admin + + def _install_galaxy(self): + """Override to skip Galaxy installation - uvx manages this.""" + # No-op for uvx since it manages Galaxy installation + return True + + def _ensure_galaxy_repository_available(self): + """Override to skip repository cloning - not needed for uvx.""" + # No-op for uvx since no repository is needed + return True + + +@contextlib.contextmanager +def uvx_galaxy_config(ctx, runnables, for_tests=False, **kwds): + """Set up a ``UvxGalaxyConfig`` in an auto-cleaned context.""" + test_data_dir = _find_test_data(runnables, **kwds) + + with _config_directory(ctx, **kwds) as config_directory: + + def config_join(*args): + return os.path.join(config_directory, *args) + + server_name = "main" + + # Ensure dependency resolvers are configured + ensure_dependency_resolvers_conf_configured(ctx, kwds, os.path.join(config_directory, "resolvers_conf.xml")) + + # Handle basic galaxy configuration without installation + galaxy_root = config_directory # Use config directory as galaxy root for uvx + # Skip refgenie config for uvx since Galaxy is managed by uvx + + # Setup tool paths (but don't require galaxy_root) + all_tool_paths = _all_tool_paths(runnables, galaxy_root=None, extra_tools=kwds.get("extra_tools")) + kwds["all_in_one_handling"] = True + _handle_job_config_file(config_directory, server_name, test_data_dir, all_tool_paths, kwds) + _handle_file_sources(config_directory, test_data_dir, kwds) + + # Basic paths setup + file_path = kwds.get("file_path") or config_join("files") + _ensure_directory(file_path) + + tool_dependency_dir = kwds.get("tool_dependency_dir") or config_join("deps") + _ensure_directory(tool_dependency_dir) + + shed_tool_conf = kwds.get("shed_tool_conf") or config_join("shed_tools_conf.xml") + empty_tool_conf = config_join("empty_tool_conf.xml") + tool_conf = config_join("tool_conf.xml") + shed_data_manager_config_file = config_join("shed_data_manager_conf.xml") + + shed_tool_path = kwds.get("shed_tool_path") or config_join("shed_tools") + _ensure_directory(shed_tool_path) + + sheds_config_path = _configure_sheds_config_file(ctx, config_directory, runnables, **kwds) + + database_location = config_join("galaxy.sqlite") + master_api_key = _get_master_api_key(kwds) + dependency_dir = os.path.join(config_directory, "deps") + _ensure_directory(dependency_dir) + port = _get_port(kwds) + + # Template args for file generation + # Use fallback for tool shed URL if none configured + shed_target_url = tool_shed_url(ctx, **kwds) or MAIN_TOOLSHED_URL + + template_args = dict( + shed_tool_path=shed_tool_path, + shed_tool_conf=shed_tool_conf, + shed_data_manager_config_file=shed_data_manager_config_file, + test_data_dir=test_data_dir, + shed_target_url=shed_target_url, + dependency_dir=dependency_dir, + file_path=file_path, + temp_directory=config_directory, + ) + + # Galaxy properties + properties = _shared_galaxy_properties(config_directory, kwds, for_tests=for_tests) + properties.update( + dict( + server_name=server_name, + host=kwds.get("host", "127.0.0.1"), + port=str(port), + enable_celery_tasks="true", + ftp_upload_dir_template="${ftp_upload_dir}", + ftp_upload_purge="false", + ftp_upload_dir=test_data_dir or os.path.abspath("."), + ftp_upload_site="Test Data", + check_upload_content="false", + tool_dependency_dir=dependency_dir, + file_path=file_path, + new_file_path="${temp_directory}/tmp", + tool_config_file=f"{tool_conf},{shed_tool_conf}", + tool_sheds_config_file=sheds_config_path, + manage_dependency_relationships="false", + job_working_directory="${temp_directory}/job_working_directory", + template_cache_path="${temp_directory}/compiled_templates", + citation_cache_type="file", + citation_cache_data_dir="${temp_directory}/citations/data", + citation_cache_lock_dir="${temp_directory}/citations/lock", + database_auto_migrate="true", + enable_beta_tool_formats="true", + id_secret="${id_secret}", + log_level="DEBUG" if ctx.verbose else "INFO", + debug="true" if ctx.verbose else "false", + watch_tools="auto", + default_job_shell="/bin/bash", + integrated_tool_panel_config=("${temp_directory}/integrated_tool_panel_conf.xml"), + migrated_tools_config=empty_tool_conf, + test_data_dir=test_data_dir, + shed_data_manager_config_file=shed_data_manager_config_file, + outputs_to_working_directory="true", + object_store_store_by="uuid", + ) + ) + + _handle_container_resolution(ctx, kwds, properties) + properties["database_connection"] = _database_connection(database_location, **kwds) + + if kwds.get("mulled_containers", False): + properties["mulled_channels"] = kwds.get("conda_ensure_channels", "") + + _handle_kwd_overrides(properties, kwds) + + # Build environment + env = _build_env_for_galaxy(properties, template_args) + env["PLANEMO"] = "1" + env["GALAXY_DEVELOPMENT_ENVIRONMENT"] = "1" + + # Write configuration files (but skip Galaxy installation) + # Assume uvx Galaxy is modern (>= 22.01) and write YAML config directly + env["GALAXY_CONFIG_FILE"] = config_join("galaxy.yml") + env["GRAVITY_STATE_DIR"] = config_join("gravity") + with NamedTemporaryFile(suffix=".sock", delete=True) as nt: + env["SUPERVISORD_SOCKET"] = nt.name + write_file( + env["GALAXY_CONFIG_FILE"], + json.dumps( + { + "galaxy": properties, + "gravity": { + "galaxy_root": galaxy_root, + "gunicorn": { + "bind": f"{kwds.get('host', 'localhost')}:{port}", + "preload": False, + }, + "gx-it-proxy": { + "enable": False, + }, + }, + }, + indent=2, + ), + ) + + # Write tool configurations + tool_definition = _tool_conf_entry_for(all_tool_paths) + write_file(tool_conf, _sub(TOOL_CONF_TEMPLATE, dict(tool_definition=tool_definition))) + + shed_tool_conf_contents = _sub(SHED_TOOL_CONF_TEMPLATE, template_args) + write_file(shed_tool_conf, shed_tool_conf_contents, force=False) + write_file(shed_data_manager_config_file, SHED_DATA_MANAGER_CONF_TEMPLATE) + + yield UvxGalaxyConfig( + ctx, + config_directory, + env, + test_data_dir, + port, + server_name, + master_api_key, + runnables, + kwds, + ) + + __all__ = ( "DATABASE_LOCATION_TEMPLATE", "galaxy_config", + "UvxGalaxyConfig", + "uvx_galaxy_config", ) diff --git a/planemo/galaxy/serve.py b/planemo/galaxy/serve.py index 58e352b40..b33f35639 100644 --- a/planemo/galaxy/serve.py +++ b/planemo/galaxy/serve.py @@ -31,6 +31,8 @@ def _serve(ctx, runnables, **kwds): engine = kwds.get("engine", "galaxy") if engine == "docker_galaxy": kwds["dockerize"] = True + elif engine == "uvx_galaxy": + kwds["uvx_galaxy"] = True daemon = kwds.get("daemon", False) if daemon: diff --git a/planemo/io.py b/planemo/io.py index 72f2dbed1..e59d30466 100644 --- a/planemo/io.py +++ b/planemo/io.py @@ -5,6 +5,7 @@ import fnmatch import os import shutil +import signal import subprocess import sys import tempfile @@ -228,18 +229,19 @@ def kill_posix(pid: int): def _check_pid(): try: - os.kill(pid, 0) + os.kill(pid, signal.SIGTERM) return True except OSError: return False if _check_pid(): - for sig in [15, 9]: + for sig in [signal.SIGTERM, signal.SIGKILL]: try: # gunicorn (unlike paste), seem to require killing process # group os.killpg(os.getpgid(pid), sig) - except OSError: + except OSError as e: + print(f"Failed to kill process group for pid {pid} with signal {sig}: {e}") return time.sleep(1) if not _check_pid(): diff --git a/planemo/options.py b/planemo/options.py index 5044a7b74..08edea551 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -70,13 +70,13 @@ def run_engine_option(): """Annotate click command as consume the --engine option.""" return planemo_option( "--engine", - type=click.Choice(["galaxy", "docker_galaxy", "cwltool", "toil", "external_galaxy"]), + type=click.Choice(["galaxy", "docker_galaxy", "cwltool", "toil", "external_galaxy", "uvx_galaxy"]), default=None, use_global_config=True, help=( "Select an engine to run or test artifacts such as tools " "and workflows. Defaults to a local Galaxy, but running Galaxy within " - "a Docker container or the CWL reference implementation 'cwltool' and " + "a Docker container, via uvx, or the CWL reference implementation 'cwltool' and " "'toil' be selected." ), ) @@ -100,14 +100,14 @@ def serve_engine_option(): """ return planemo_option( "--engine", - type=click.Choice(["galaxy", "docker_galaxy", "external_galaxy"]), + type=click.Choice(["galaxy", "docker_galaxy", "external_galaxy", "uvx_galaxy"]), default="galaxy", use_global_config=True, use_env_var=True, help=( "Select an engine to serve artifacts such as tools " "and workflows. Defaults to a local Galaxy, but running Galaxy within " - "a Docker container." + "a Docker container or via uvx is also supported." ), ) diff --git a/planemo/shed/__init__.py b/planemo/shed/__init__.py index 1082267fa..e3b8b76f7 100644 --- a/planemo/shed/__init__.py +++ b/planemo/shed/__init__.py @@ -792,7 +792,7 @@ def shed_repo_type(config, name): def _shed_config_to_url(shed_config): url = shed_config["url"] - if not url.startswith("http"): + if url and not url.startswith("http"): message = f"Invalid shed url specified [{url}]. Please specify a valid HTTP address or one of {list(SHED_SHORT_NAMES.keys())}" raise ValueError(message) return url