Merge pull request #523 from abretaud/1905fix

19.05 update conda + fix slurm not starting

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "galaxy/roles/galaxyproject.galaxyextras"]
 	path = galaxy/roles/galaxyprojectdotorg.galaxyextras
 	url = https://github.com/galaxyproject/ansible-galaxy-extras
-        branch = 18.09
+        branch = 19.05
 [submodule "galaxy/roles/galaxy-postgresql"]
 	path = galaxy/roles/galaxy-postgresql
 	url = https://github.com/galaxyproject/ansible-postgresql

README.md

@@ -6,21 +6,21 @@
 [![docker image stats](https://images.microbadger.com/badges/image/bgruening/galaxy-stable.svg)](https://microbadger.com/images/bgruening/galaxy-stable "Get your own image badge on microbadger.com")
 
 
-:information_source: `19.01` will be the last release which is based on `ubuntu:14.04` and
-PostgreSQL 9.3. We will migrate to `ubuntu:18.04` and a newer PostgreSQL version in `19.05`. Furthermore, we will not
-support old Galaxy tool dependencies. These are deprecated since a few years and we think its time to remove support
+:information_source: `19.01` was the last release which is based on `ubuntu:14.04` and
+PostgreSQL 9.3. In 19.05 we have migrate to `ubuntu:18.04` and PostgreSQL version 11.5. Furthermore, we no longer
+support old Galaxy tool dependencies. These are deprecated since a few years and we think it's time to remove support
 for this in the default installation. You can install all needed packages manually to enable support again, though.
 
 In short, with 19.05:
-  * Galaxy tool dependencies will not be supported by default
-  * `ubuntu:18.04` will be the new base image
-  * a new PostgreSQL version means you need to migrate any old database to the new version
+  * Galaxy tool dependencies are be supported by default
+  * `ubuntu:18.04` is the new base image
+  * a new PostgreSQL version means you need to [migrate any old database to the new version](#Postgresql-migration)
 
 
 Galaxy Docker Image
 ===================
 
-The [Galaxy](http://www.galaxyproject.org) [Docker](http://www.docker.io) Image is an easy distributable full-fledged Galaxy installation, that can be used for testing, teaching and presenting new tools and features. 
+The [Galaxy](http://www.galaxyproject.org) [Docker](http://www.docker.io) Image is an easy distributable full-fledged Galaxy installation, that can be used for testing, teaching and presenting new tools and features.
 
 One of the main goals is to make the access to entire tool suites as easy as possible. Usually,
 this includes the setup of a public available web-service that needs to be maintained, or that the Tool-user needs to either setup a Galaxy Server by its own or to have Admin access to a local Galaxy server.
@@ -34,6 +34,7 @@ The Image is based on [Ubuntu 14.04 LTS](http://releases.ubuntu.com/14.04/) and
 
 - [Usage](#Usage)
   - [Upgrading images](#Upgrading-images)
+    - [Upgrading images](#Postgresql-migration)
   - [Enabling Interactive Environments in Galaxy](#Enabling-Interactive-Environments-in-Galaxy)
   - [Using passive mode FTP or SFTP](#Using-passive-mode-FTP-or-SFTP)
   - [Using Parent docker](#Using-Parent-docker)
@@ -218,6 +219,59 @@ We will release a new version of this image concurrent with every new Galaxy rel
 13. Clean the old container and image
 
 
+### Postgresql migration <a name="Postgresql-migration" /> [[toc]](#toc)
+
+In the 19.05 version, Postgresql was updated from version 9.3 to version 11.5. If you are upgrading from a version <19.05, you will need to migrate the database.
+You can do it the following way for example:
+
+1. Stop Galaxy in the old container
+
+```sh
+docker exec <old_container_name> supervisorctl stop galaxy:
+```
+
+2. Dump the old database
+
+```sh
+docker exec <old_container_name> bash
+su postgres
+pg_dumpall --clean > /export/postgresql/9.3dump.sql
+exit
+exit
+```
+
+3. Update the container
+
+```sh
+docker stop <old_container_name>
+docker pull bgruening/galaxy-stable
+docker run -p 8080:80 -v /data/galaxy-data:/export --name <new_container_name> bgruening/galaxy-stable
+```
+
+4. Restore the dump to the new postgres version
+
+Wait for the startup process to finish (Galaxy should be accessible)
+
+```sh
+docker exec <new_container_name> bash
+supervisorctl stop galaxy:
+su postgres
+psql -f /export/postgresql/9.3dump.sql postgres
+#Upgrade the database to the most recent version
+sh manage_db.sh upgrade
+exit
+exit
+```
+5. Restart Galaxy
+
+```sh
+docker exec <old_container_name> supervisorctl start galaxy:
+```
+
+6. Clean old files
+
+If you are *very* sure that everything went well, you can delete `/export/postgresql/9.3dump.sql` and `/export/postgresql/9.3/` to save some space.
+
 ## Enabling Interactive Environments in Galaxy <a name="Enabling-Interactive-Environments-in-Galaxy" /> [[toc]](#toc)
 
 Interactive Environments (IE) are sophisticated ways to extend Galaxy with powerful services, like [Jupyter](http://jupyter.org/), in a secure and reproducible way.
@@ -323,7 +377,7 @@ id_secret: d5c910cc6e32cad08599987ab64dcfae
 
 You should change all three configuration variables above manually in `/export/galaxy-central/config/galaxy.yml`.
 
-Alternatively you can pass the security configuration when running the image but please note that it is a security problem. E.g. if a tool exposes all `env`'s your secret API key will also be exposed. 
+Alternatively you can pass the security configuration when running the image but please note that it is a security problem. E.g. if a tool exposes all `env`'s your secret API key will also be exposed.
 
 ## Configuring Galaxy's behind a proxy <a name="Galaxy-behind-proxy" /> [[toc]](#toc)
 
@@ -839,7 +893,7 @@ If you simply want to change the Galaxy repository and/or the Galaxy branch, fro
    - tracking the Galaxy release_18.01 branch
    - uwsgi work to adopt to changes for 18.01
    - remove nodejs-legacy & npm from Dockerfile and install latest version from ansible-extras
-   - initial galaxy.ini → galaxy.yml integration 
+   - initial galaxy.ini → galaxy.yml integration
    - grafana and influxdb container (compose)
    - Galaxy telegraf integration to push to influxdb (compose)
    - added some documentation (compose)
@@ -850,9 +904,14 @@ If you simply want to change the Galaxy repository and/or the Galaxy branch, fro
    - a lot of bug-fixes to the compose setup by @abretaud
 - 19.01:
    - This is featuring the latest and greatest from the Galaxy community
-   - Please note that this release will be the last release which is based on `ubuntu:14.04` and PostgreSQL 9.3. 
+   - Please note that this release will be the last release which is based on `ubuntu:14.04` and PostgreSQL 9.3.
      We will migrate to `ubuntu:18.04` and a newer PostgreSQL version in `19.05`. Furthermore, we will not
-     support old Galaxy tool dependencies. 
+     support old Galaxy tool dependencies.
+- 19.05:
+   - The image is now based on `ubuntu:18.04` (instead of ubuntu:14.04 previously) and PostgreSQL 11.5 (9.3 previously).
+     See [migration documention](#Postgresql-migration) to migrate the postgresql database from 9.3 to 11.5.
+   - We not longer support old Galaxy tool dependencies.
+
 
 # Support & Bug Reports <a name="Support-Bug-Reports" /> [[toc]](#toc)
 

compose/galaxy-web/Dockerfile

@@ -26,7 +26,7 @@ ENV GALAXY_DESTINATIONS_DEFAULT=slurm_cluster \
     SUPERVISOR_MANAGE_REPORTS=True \
     SUPERVISOR_MANAGE_IE_PROXY=True \
     SUPERVISOR_MANAGE_CONDOR=True \
-    SUPERVISOR_MANAGE_SLURM=True \
+    SUPERVISOR_MANAGE_SLURM= \
     HOST_DOCKER_LEGACY=True
 
 RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \

galaxy/Dockerfile

@@ -117,7 +117,7 @@ ADD ./bashrc $GALAXY_HOME/.bashrc
 # Install miniconda, then virtualenv from conda and then
 # download latest stable release of Galaxy.
 
-RUN curl -s -L https://repo.continuum.io/miniconda/Miniconda2-4.5.12-Linux-x86_64.sh > ~/miniconda.sh \
+RUN curl -s -L https://repo.continuum.io/miniconda/Miniconda2-4.7.10-Linux-x86_64.sh > ~/miniconda.sh \
     && /bin/bash ~/miniconda.sh -b -p $GALAXY_CONDA_PREFIX/ \
     && rm ~/miniconda.sh \
     && ln -s $GALAXY_CONDA_PREFIX/etc/profile.d/conda.sh /etc/profile.d/conda.sh \
@@ -221,7 +221,6 @@ RUN mkdir -p /shed_tools $EXPORT_DIR/ftp/ \
     && sh $GALAXY_ROOT/create_db.sh -c "$GALAXY_CONFIG_FILE" \
     && python /usr/local/bin/create_galaxy_user.py --user $GALAXY_DEFAULT_ADMIN_USER --password $GALAXY_DEFAULT_ADMIN_PASSWORD -c $GALAXY_CONFIG_FILE --key $GALAXY_DEFAULT_ADMIN_KEY \
     && service postgresql stop \
-    && apt-get purge -y ansible \
     && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ \
     # cleanup dance
     && find $GALAXY_ROOT/ -name '*.pyc' -delete | true \
@@ -231,6 +230,10 @@ RUN mkdir -p /shed_tools $EXPORT_DIR/ftp/ \
     && find $GALAXY_VIRTUAL_ENV -name '*.pyc' -delete | true \
     && rm -rf /tmp/* /root/.cache/ /var/cache/* $GALAXY_ROOT/client/node_modules/ $GALAXY_VIRTUAL_ENV/src/ /home/galaxy/.cache/ /home/galaxy/.npm
 
+RUN touch /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog && \
+    mkdir -p /var/run/condor/ /var/lock/condor/ && \
+    chown -R condor: /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog /var/run/condor/ /var/lock/condor/
+
 # The following commands will be executed as the galaxy user
 USER $GALAXY_USER
 
@@ -332,7 +335,7 @@ ENV SUPERVISOR_POSTGRES_AUTOSTART=True \
     SUPERVISOR_MANAGE_REPORTS=True \
     SUPERVISOR_MANAGE_IE_PROXY=True \
     SUPERVISOR_MANAGE_CONDOR=True \
-    SUPERVISOR_MANAGE_SLURM=True \
+    SUPERVISOR_MANAGE_SLURM= \
     HOST_DOCKER_LEGACY= \
     GALAXY_EXTRAS_CONFIG_POSTGRES=True \
     STARTUP_EXPORT_USER_FILES=True

galaxy/startup.sh

@@ -113,6 +113,11 @@ if [[ ! -z $STARTUP_EXPORT_USER_FILES ]]; then
     python /usr/local/bin/export_user_files.py $PG_DATA_DIR_DEFAULT
 fi
 
+# Delete compiled templates in case they are out of date
+if [[ ! -z $GALAXY_CONFIG_TEMPLATE_CACHE_PATH ]]; then
+    rm -rf $GALAXY_CONFIG_TEMPLATE_CACHE_PATH/*
+fi
+
 # Enable loading of dependencies on startup. Such as LDAP.
 # Adapted from galaxyproject/galaxy/scripts/common_startup.sh
 if [[ ! -z $LOAD_GALAXY_CONDITIONAL_DEPENDENCIES ]]
@@ -420,21 +425,21 @@ if [[ ! -z $GALAXY_DEFAULT_ADMIN_USER ]]
         if [ -x /export/post-start-actions.sh ]
             then
            # uses ephemeris, present in docker-galaxy-stable, to wait for the local instance
-           galaxy-wait -g http://127.0.0.1 -v --timeout 120 > $GALAXY_LOGS_DIR/post-start-actions.log &&
+           /tool_deps/_conda/bin/galaxy-wait -g http://127.0.0.1 -v --timeout 120 > $GALAXY_LOGS_DIR/post-start-actions.log &&
            /export/post-start-actions.sh >> $GALAXY_LOGS_DIR/post-start-actions.log &
     fi
 fi
 
 # Reinstall tools if the user want to
 if [[ ! -z $GALAXY_AUTO_UPDATE_TOOLS ]]
     then
-        galaxy-wait -g http://127.0.0.1 -v --timeout 120 > /home/galaxy/logs/post-start-actions.log &&
+        /tool_deps/_conda/bin/galaxy-wait -g http://127.0.0.1 -v --timeout 120 > /home/galaxy/logs/post-start-actions.log &&
         OLDIFS=$IFS
         IFS=','
         for TOOL_YML in `echo "$GALAXY_AUTO_UPDATE_TOOLS"`
         do
             echo "Installing tools from $TOOL_YML"
-            shed-tools install -g "http://127.0.0.1" -a "$GALAXY_DEFAULT_ADMIN_KEY" -t "$TOOL_YML"
+            /tool_deps/_conda/bin/shed-tools install -g "http://127.0.0.1" -a "$GALAXY_DEFAULT_ADMIN_KEY" -t "$TOOL_YML"
             /tool_deps/_conda/bin/conda clean --tarballs --yes
         done
         IFS=$OLDIFS