Various fixes for targeting postgres in singularity

planemo/database/factory.py

@@ -1,5 +1,7 @@
 """Create a DatabaseSource from supplied planemo configuration."""
 
+from typing import Optional
+
 from galaxy.util.commands import which
 
 from .interface import DatabaseSource
@@ -8,7 +10,7 @@
 from .postgres_singularity import SingularityPostgresDatabaseSource
 
 
-def create_database_source(**kwds) -> DatabaseSource:
+def create_database_source(profile_directory: Optional[str] = None, **kwds) -> DatabaseSource:
     """Return a :class:`planemo.database.interface.DatabaseSource` for configuration."""
     database_type = kwds.get("database_type", "auto")
     if database_type == "auto":
@@ -26,7 +28,7 @@ def create_database_source(**kwds) -> DatabaseSource:
     elif database_type == "postgres_docker":
         return DockerPostgresDatabaseSource(**kwds)
     elif database_type == "postgres_singularity":
-        return SingularityPostgresDatabaseSource(**kwds)
+        return SingularityPostgresDatabaseSource(profile_directory=profile_directory, **kwds)
     # TODO
     # from .sqlite import SqliteDatabaseSource
     # elif database_type == "sqlite":

planemo/database/interface.py

@@ -1,18 +1,12 @@
 """Describe the interface classes of the planemo.database package."""
 
 import abc
+from typing import Optional
 
 
 class DatabaseSource(metaclass=abc.ABCMeta):
     """Interface describing a source of profile databases."""
 
-    @abc.abstractmethod
-    def create_database(self, identifier):
-        """Create a database with specified short identifier.
-
-        Throw an exception if it already exists.
-        """
-
     @abc.abstractmethod
     def delete_database(self, identifier):
         """Delete a database with specified short identifier.
@@ -25,8 +19,16 @@ def list_databases(self):
         """Return identifiers associated with database source."""
 
     @abc.abstractmethod
-    def sqlalchemy_url(self, identifier):
+    def sqlalchemy_url(self, identifier) -> Optional[str]:
         """Return a URL string for use by sqlalchemy."""
 
+    def start(self):
+        """Start the database source, if necessary."""
+        pass
+
+    def stop(self):
+        """Stop the database source, if necessary."""
+        pass
+
 
 __all__ = ("DatabaseSource",)

planemo/database/postgres_docker.py

@@ -63,11 +63,25 @@ def __init__(self, **kwds):
         self.database_port = DEFAULT_POSTGRES_PORT_EXPOSE
         self._kwds = kwds
         self._docker_host_kwds = dockerfiles.docker_host_args(**kwds)
+
+    def create_database(self, identifier):
+        # Not needed for profile creation, database will be created when Galaxy starts.
+        pass
+
+    def delete_database(self, identifier):
+        # Not needed for profile deletion, just remove the profile directory.
+        pass
+
+    def start(self):
         if not is_running_container(**self._docker_host_kwds):
             start_postgres_docker(**self._docker_host_kwds)
             # Hack to give docker a bit of time to boot up and allow psql to start.
             time.sleep(30)
 
+    def stop(self):
+        if is_running_container(**self._docker_host_kwds):
+            stop_postgres_docker(**self._docker_host_kwds)
+
     def sqlalchemy_url(self, identifier):
         """Return URL or form postgresql://username:password@localhost/mydatabase."""
         return "postgresql://%s:%s@%s:%d/%s" % (

planemo/database/postgres_singularity.py

@@ -1,11 +1,12 @@
 """Module describes a :class:`DatabaseSource` for managed, dockerized postgres databases."""
 
 import os
-import subprocess
+import shutil
 import time
 from tempfile import mkdtemp
+from typing import Optional
 
-from galaxy.util.commands import execute
+from galaxy.util.commands import shell_process
 
 from planemo.io import info
 from .interface import DatabaseSource
@@ -15,21 +16,15 @@
 DEFAULT_POSTGRES_DATABASE_NAME = "galaxy"
 DEFAULT_POSTGRES_USER = "galaxy"
 DEFAULT_POSTGRES_PASSWORD = "mysecretpassword"
-DEFAULT_POSTGRES_PORT_EXPOSE = 5432
 DEFAULT_DOCKERIMAGE = "postgres:14.2-alpine3.15"
-DEFAULT_SIF_NAME = "postgres_14_2-alpine3_15.sif"
-
-DEFAULT_CONNECTION_STRING = f"postgresql://{DEFAULT_POSTGRES_USER}:{DEFAULT_POSTGRES_PASSWORD}@localhost:{DEFAULT_POSTGRES_PORT_EXPOSE}/{DEFAULT_POSTGRES_DATABASE_NAME}"
 
 
 def start_postgres_singularity(
     singularity_path,
-    container_instance_name,
     database_location,
     databasename=DEFAULT_POSTGRES_DATABASE_NAME,
     user=DEFAULT_POSTGRES_USER,
     password=DEFAULT_POSTGRES_PASSWORD,
-    **kwds,
 ):
     info(f"Postgres database stored at: {database_location}")
     pgdata_path = os.path.join(database_location, "pgdata")
@@ -40,66 +35,41 @@ def start_postgres_singularity(
     if not os.path.exists(pgrun_path):
         os.makedirs(pgrun_path)
 
-    version_file = os.path.join(pgdata_path, "PG_VERSION")
-    if not os.path.exists(version_file):
-        # Run container for a short while to initialize the database
-        # The database will not be initilizaed during a
-        # "singularity instance start" command
-        init_database_command = [
-            singularity_path,
-            "run",
-            "-B",
-            f"{pgdata_path}:/var/lib/postgresql/data",
-            "-B",
-            f"{pgrun_path}:/var/run/postgresql",
-            "-e",
-            "-C",
-            "--env",
-            f"POSTGRES_DB={databasename}",
-            "--env",
-            f"POSTGRES_USER={user}",
-            "--env",
-            f"POSTGRES_PASSWORD={password}",
-            "--env",
-            "POSTGRES_INITDB_ARGS='--encoding=UTF-8'",
-            f"docker://{DEFAULT_DOCKERIMAGE}",
-        ]
-        info(f"Initilizing postgres database in folder: {pgdata_path}")
-        process = subprocess.Popen(init_database_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        # Give the container time to initialize the database
-        for _ in range(10):
-            if os.path.exists(version_file):
-                break
-            time.sleep(5)
-            info("Waiting for the postgres database to initialize.")
-        else:
-            raise Exception("Failed to initialize the postgres database.")
-        time.sleep(10)
-        process.terminate()
-
-    # Start the singularity instance, assumes the database is
-    # already initialized since the entrypoint will not be run
-    # when starting a instance of the container.
+    VERSION_FILE = os.path.join(pgdata_path, "PG_VERSION")
     run_command = [
         singularity_path,
-        "instance",
-        "start",
+        "run",
         "-B",
         f"{pgdata_path}:/var/lib/postgresql/data",
         "-B",
         f"{pgrun_path}:/var/run/postgresql",
         "-e",
         "-C",
+        "--env",
+        f"POSTGRES_DB={databasename}",
+        "--env",
+        f"POSTGRES_USER={user}",
+        "--env",
+        f"POSTGRES_PASSWORD={password}",
+        "--env",
+        "POSTGRES_INITDB_ARGS='--encoding=UTF-8'",
         f"docker://{DEFAULT_DOCKERIMAGE}",
-        container_instance_name,
     ]
-    info(f"Starting singularity instance named: {container_instance_name}")
-    execute(run_command)
-
-
-def stop_postgress_singularity(container_instance_name, **kwds):
-    info(f"Stopping singularity instance named: {container_instance_name}")
-    execute(["singularity", "instance", "stop", container_instance_name])
+    info("Starting postgres singularity container")
+    p = shell_process(run_command)
+    # Give the container time to initialize the database
+    for _ in range(10):
+        if os.path.exists(VERSION_FILE):
+            break
+        time.sleep(5)
+        info("Waiting for the postgres database to initialize.")
+    else:
+        try:
+            p.terminate()
+        except Exception as e:
+            info(f"Failed to terminate process: {e}")
+        raise Exception("Failed to initialize the postgres database.")
+    return p
 
 
 class SingularityPostgresDatabaseSource(ExecutesPostgresSqlMixin, DatabaseSource):
@@ -108,42 +78,58 @@ class SingularityPostgresDatabaseSource(ExecutesPostgresSqlMixin, DatabaseSource
     "with" statements to automatically start and stop the container.
     """
 
-    def __init__(self, **kwds):
+    def __init__(self, profile_directory: Optional[str] = None, **kwds):
         """Construct a postgres database source from planemo configuration."""
 
         self.singularity_path = "singularity"
         self.database_user = DEFAULT_POSTGRES_USER
         self.database_password = DEFAULT_POSTGRES_PASSWORD
-        self.database_host = "localhost"  # TODO: Make docker host
-        self.database_port = DEFAULT_POSTGRES_PORT_EXPOSE
-        if "postgres_storage_location" in kwds and kwds["postgres_storage_location"] is not None:
+        if kwds.get("postgres_storage_location") is not None:
             self.database_location = kwds["postgres_storage_location"]
+        elif profile_directory:
+            self.database_location = os.path.join(profile_directory, "postgres")
         else:
             self.database_location = os.path.join(mkdtemp(suffix="_planemo_postgres_db"))
+        self.database_socket_dir = os.path.join(self.database_location, "pgrun")
         self.container_instance_name = f"{DEFAULT_CONTAINER_NAME}-{int(time.time() * 1000000)}"
         self._kwds = kwds
+        self.running_process = None
+
+    def create_database(self, identifier):
+        # Not needed, we'll create the database automatically when the container starts.
+        pass
+
+    def delete_database(self, identifier):
+        shutil.rmtree(self.database_location, ignore_errors=True)
 
-    def __enter__(self):
-        start_postgres_singularity(
+    def start(self):
+        self.running_process = start_postgres_singularity(
             singularity_path=self.singularity_path,
             database_location=self.database_location,
             user=self.database_user,
             password=self.database_password,
-            container_instance_name=self.container_instance_name,
-            **self._kwds,
         )
 
-    def __exit__(self, exc_type, exc_value, traceback):
-        stop_postgress_singularity(self.container_instance_name)
+    def stop(self):
+        if self.running_process:
+            try:
+                self.running_process.terminate()
+                postmaster_pid_file = os.path.join(self.database_location, "pgdata", "postmaster.pid")
+                if os.path.exists(postmaster_pid_file):
+                    os.remove(postmaster_pid_file)
+                postmaster_lock_file = os.path.join(self.database_location, "pgrun", ".s.PGSQL.5432.lock")
+                if os.path.exists(postmaster_lock_file):
+                    os.remove(postmaster_lock_file)
+            except Exception as e:
+                info(f"Failed to terminate process: {e}")
 
     def sqlalchemy_url(self, identifier):
-        """Return URL or form postgresql://username:password@localhost/mydatabase."""
-        return "postgresql://%s:%s@%s:%d/%s" % (
+        """Return URL for PostgreSQL connection via Unix socket."""
+        return "postgresql://%s:%s@/%s?host=%s" % (
             self.database_user,
             self.database_password,
-            self.database_host,
-            self.database_port,
             identifier,
+            self.database_socket_dir,
         )
 
 

planemo/engine/factory.py

@@ -9,7 +9,6 @@
     DockerizedManagedGalaxyEngine,
     ExternalGalaxyEngine,
     LocalManagedGalaxyEngine,
-    LocalManagedGalaxyEngineWithSingularityDB,
 )
 from .toil import ToilEngine
 
@@ -26,10 +25,7 @@ def build_engine(ctx, **kwds):
     """Build an engine from the supplied planemo configuration."""
     engine_type_str = kwds.get("engine", "galaxy")
     if engine_type_str == "galaxy":
-        if "database_type" in kwds and kwds["database_type"] == "postgres_singularity":
-            engine_type = LocalManagedGalaxyEngineWithSingularityDB
-        else:
-            engine_type = LocalManagedGalaxyEngine
+        engine_type = LocalManagedGalaxyEngine
     elif engine_type_str == "docker_galaxy":
         engine_type = DockerizedManagedGalaxyEngine
     elif engine_type_str == "external_galaxy":

planemo/engine/galaxy.py

@@ -12,7 +12,6 @@
 from galaxy.tool_util.verify import interactor
 
 from planemo import io
-from planemo.database.postgres_singularity import SingularityPostgresDatabaseSource
 from planemo.galaxy.activity import (
     execute,
     execute_rerun,
@@ -172,13 +171,6 @@ def _serve_kwds(self):
         return self._kwds.copy()
 
 
-class LocalManagedGalaxyEngineWithSingularityDB(LocalManagedGalaxyEngine):
-    def run(self, runnables, job_paths, output_collectors: Optional[List[Callable]] = None):
-        with SingularityPostgresDatabaseSource(**self._kwds.copy()):
-            run_responses = super().run(runnables, job_paths, output_collectors)
-        return run_responses
-
-
 class DockerizedManagedGalaxyEngine(LocalManagedGalaxyEngine):
     """An :class:`Engine` implementation backed by Galaxy running in Docker.
 
@@ -233,5 +225,4 @@ def expand_test_cases(config, test_cases):
     "DockerizedManagedGalaxyEngine",
     "ExternalGalaxyEngine",
     "LocalManagedGalaxyEngine",
-    "LocalManagedGalaxyEngineWithSingularityDB",
 )