Add logic for upgrading datadir

Based on the MYSQL_UPGRADE variable and based on the datadir version detected,
we either warn user about incompatibilities, or run one of the specified
tools:
* mysql_upgrade
* mysqlcheck --analyze --all-databases
* mysqlcheck --optimize --all-databases

This should also fix sclorg#33

10.1/root/usr/bin/run-mysqld

@@ -26,6 +26,10 @@ if [ -f ${CONTAINER_SCRIPTS_PATH}/post-init.sh ]; then
   log_info 'Sourcing post-init.sh ...'
   source ${CONTAINER_SCRIPTS_PATH}/post-init.sh
 fi
+if [ -f ${CONTAINER_SCRIPTS_PATH}/check-upgrade.sh ]; then
+  log_info 'Sourcing check-upgrade.sh ...'
+  source ${CONTAINER_SCRIPTS_PATH}/check-upgrade.sh
+fi
 
 # Restart the MySQL server with public IP bindings
 shutdown_local_mysql

10.1/root/usr/bin/run-mysqld-master

@@ -39,8 +39,14 @@ mysql $mysql_flags <<EOSQL
   FLUSH PRIVILEGES;
 EOSQL
 
-log_info 'Sourcing post-init.sh ...'
-[ -f ${CONTAINER_SCRIPTS_PATH}/post-init.sh ] && source ${CONTAINER_SCRIPTS_PATH}/post-init.sh
+if [ -f ${CONTAINER_SCRIPTS_PATH}/post-init.sh ]; then
+  log_info 'Sourcing post-init.sh ...'
+  source ${CONTAINER_SCRIPTS_PATH}/post-init.sh
+fi
+if [ -f ${CONTAINER_SCRIPTS_PATH}/check-upgrade.sh ]; then
+  log_info 'Sourcing check-upgrade.sh ...'
+  source ${CONTAINER_SCRIPTS_PATH}/check-upgrade.sh
+fi
 
 # Restart the MySQL server with public IP bindings
 shutdown_local_mysql

10.1/root/usr/bin/run-mysqld-slave

@@ -48,8 +48,14 @@ mysql $mysql_flags <<EOSQL
   START SLAVE;
 EOSQL
 
-log_info 'Sourcing post-init.sh ...'
-[ -f ${CONTAINER_SCRIPTS_PATH}/post-init.sh ] && source ${CONTAINER_SCRIPTS_PATH}/post-init.sh
+if [ -f ${CONTAINER_SCRIPTS_PATH}/post-init.sh ]; then
+  log_info 'Sourcing post-init.sh ...'
+  source ${CONTAINER_SCRIPTS_PATH}/post-init.sh
+fi
+if [ -f ${CONTAINER_SCRIPTS_PATH}/check-upgrade.sh ]; then
+  log_info 'Sourcing check-upgrade.sh ...'
+  source ${CONTAINER_SCRIPTS_PATH}/check-upgrade.sh
+fi
 
 # Restart the MySQL server with public IP bindings
 shutdown_local_mysql

10.1/root/usr/share/container-scripts/mysql/README.md

@@ -124,6 +124,42 @@ variable with the full path of the file you wish to use. For example, the defaul
 location is `/etc/my.cnf` but you can change it to `/etc/mysql/my.cnf` by setting
  `MYSQL_DEFAULTS_FILE=/etc/mysql/my.cnf`
 
+Upgrading and datadir checking
+------------------------------
+MySQL and MariaDB use versions that consist of three numbers (e.g. 5.6.23). When two
+versions are different only in the last part, they are compatible, but when upgrading
+from e.g. 5.6 to 5.7 or from 10.0 to 10.1, we need to consider upgrading as described
+in https://dev.mysql.com/doc/refman/5.7/en/upgrading-from-previous-series.html.
+
+Important: Upgrading to a new version is always risky and users are expected to make a full
+back-up of all data before.
+
+A safer solution to upgrade is to dump all data using `mysqldump` or `mysqldbexport` and then
+load the data using `mysql` or `mysqldbimport` into an empty (freshly initialized) database.
+
+Another way of proceeding with the upgrade is starting the new version of the `mysqld` daemon
+and run `mysql_upgrade` right after the start. This so called in-place upgrade is only possible
+if upgrading from the very previous version, so skipping versions is not possible.
+
+This container tries to detect whether the data needs to be upgraded using `mysql_upgrade` and
+we can control it by setting `MYSQL_UPGRADE` variable, which can have the following values:
+
+ * `warn` -- If the data version can be determined and the data come from a different version
+   of the daemon, a warning is printed but the container starts. This is the default value.
+ * `auto` -- `mysql_upgrade` is run at the beginning of the container start, if the data version
+   can be determined and the data come with the very previous version. A warning is printed if the data
+   come from even older or newer version. This value effectively enables automatic upgrades,
+   but it is always risky and users should still back-up all the data before starting the newer container.
+   Set this option only if you have very good back-ups at any moment and you are fine to fail-over
+   from the back-up.
+ * `force` -- `mysql_upgrade` is run right after the daemon has started, no matter what version the data
+   come from. This can be also used to creating the `mysql_upgrade_info` file if not present in the root
+   of the data directory; this file holds information about the version of the data.
+ * `optimize` -- runs `mysqlcheck --optimize` before starting the mysqld daemon, no matter what version
+   of the data is detected. It optimizes all the tables.
+ * `analyze` -- runs `mysqlcheck --analyze` before starting the mysqld daemon, no matter what version
+   of the data is detected. It analyzes all the tables.
+
 Changing the replication binlog_format
 --------------------------------------
 Some applications may wish to use `row` binlog_formats (for example, those built

10.1/root/usr/share/container-scripts/mysql/check-upgrade.sh

@@ -0,0 +1,47 @@
+check_datadir_version() {
+  local datadir="$1"
+  local datadir_version=$(get_datadir_version "$datadir")
+  local mysqld_version=$(mysqld_compat_version)
+
+  case "${MYSQL_UPGRADE}" in
+    auto|warn)
+      if [ "${datadir_version}" -eq "${mysqld_version}" ] ; then
+        log_info 'MySQL server version check OK.'
+        return 0
+      fi
+
+      if [ $(( ${datadir_version} + 1 )) -eq "${mysqld_version}" -o "${datadir_version}" -eq 505 -a "${mysqld_version}" -eq 1000 ] ; then
+        log_info "MySQL server is version '${mysqld_version}' and datadir is version '${datadir_version}'."
+        if [ "${MYSQL_UPGRADE}" == 'auto' ] ; then
+          log_info 'Running mysql_upgrade ...'
+          mysql_upgrade --socket=/tmp/mysql.sock
+        fi
+      else
+        log_info "MySQL server is version '${mysqld_version}' and datadir is version '${datadir_version}', which are incompatible."
+      fi
+      ;;
+
+    force)
+      log_info 'Running mysql_upgrade --force ...'
+      mysql_upgrade --socket=/tmp/mysql.sock --force
+      ;;
+
+    optimize)
+      log_info 'Running mysqlcheck --optimize --all-databases ...'
+      mysqlcheck --socket=/tmp/mysql.sock --optimize --all-databases
+      ;;
+
+    analyze)
+      log_info 'Running mysqlcheck --analyze --all-databases ...'
+      mysqlcheck --socket=/tmp/mysql.sock --analyze --all-databases
+      ;;
+    *)
+      log_info "Unknown value of MYSQL_UPGRADE variable: '${MYSQL_UPGRADE}', ignoring."
+      ;;
+  esac
+}
+
+check_datadir_version "${MYSQL_DATADIR}"
+
+unset -f run_upgrade check_datadir_version
+

10.1/root/usr/share/container-scripts/mysql/common.sh

@@ -38,6 +38,8 @@ export MYSQL_INNODB_BUFFER_POOL_SIZE=${MYSQL_INNODB_BUFFER_POOL_SIZE:-$innodb_bu
 export MYSQL_INNODB_LOG_FILE_SIZE=${MYSQL_INNODB_LOG_FILE_SIZE:-$innodb_log_file_size}
 export MYSQL_INNODB_LOG_BUFFER_SIZE=${MYSQL_INNODB_LOG_BUFFER_SIZE:-$innodb_log_buffer_size}
 
+export MYSQL_UPGRADE=${MYSQL_UPGRADE:-warn}
+
 # Be paranoid and stricter than we should be.
 # https://dev.mysql.com/doc/refman/en/identifiers.html
 mysql_identifier_regex='^[a-zA-Z0-9_]+$'
@@ -92,6 +94,7 @@ function initialize_database() {
   # Using empty --basedir to work-around https://bugzilla.redhat.com/show_bug.cgi?id=1406391
   mysql_install_db --rpm --datadir=$MYSQL_DATADIR --basedir=''
   start_local_mysql "$@"
+  mysql_upgrade --socket=/tmp/mysql.sock
 
   if [ -v MYSQL_RUNNING_AS_SLAVE ]; then
     log_info 'Initialization finished'
@@ -162,3 +165,63 @@ function wait_for_mysql_master() {
     sleep 1
   done
 }
+
+# Converts string version to the integer format (5.5 is converted to 505,
+# 10.1 is converted into 1001, etc.
+function version2number() {
+  local version=0
+  local version_text="$1"
+  local version_major=$(echo "$version_text" | sed -e 's/\([0-9]*\)\.\([0-9]*\)\..*$/\1/')
+  local version_minor=$(echo "$version_text" | sed -e 's/\([0-9]*\)\.\([0-9]*\)\..*$/\2/')
+  if [[ $version_major =~ ^[0-9]+$ ]] && [[ $version_minor =~ ^[0-9]+$ ]] ; then
+    version=$((version_major*100+version_minor))
+  fi
+  if [ "$version" -eq 0 ] ; then
+    echo "Error: version '$version_text' cannot be converted to the number."
+    echo "0"
+  fi
+  echo "${version}"
+}
+
+# Prints version of the mysqld that is currently available (string)
+function mysqld_version() {
+  ${MYSQL_PREFIX}/libexec/mysqld -V | awk '{print $3}'
+}
+
+# Returns version from the daemon in integer format
+function mysqld_compat_version() {
+  local compat_version=$(mysqld_version)
+  echo $(version2number $(mysqld_version))
+}
+
+# Returns version from the datadir
+function get_datadir_version() {
+  local datadir="$1"
+  local upgrade_info_file=$(get_mysql_upgrade_info_file "$datadir")
+  local version_text=$(cat "$upgrade_info_file" | head -n 1)
+  local version=$(version2number "${version_text}")
+
+  # version of the data could not be determined nor guessed
+  if [ "$version" -eq 0 ] ; then
+    local mysqld_version=$(mysqld_version)
+    echo "Warning: Version of the data could not be determined. Running such a container is very risky. The current daemon version is ${mysqld_version}. If you are sure that the data dir format corresponds with this version, create a file called 'mysql_upgrade_info' that includes just the string '${mysqld_version}' in the root of the data directory or run 'mysql_upgrade' utility that creates such a file as well. That will enable correct upgrade check in the future."
+    return 1
+  fi
+
+  echo "${version}"
+}
+
+# Returns name of the file in the datadir that holds version information about the data
+function get_mysql_upgrade_info_file() {
+  local datadir="$1"
+  echo "$datadir/mysql_upgrade_info"
+}
+
+# Gets version of the current daemon and stores it into the appropriate file in the datadir
+function store_mysqld_version() {
+  local datadir="$1"
+  local version=$(mysqld_version)
+  local upgrade_info_file=$(get_mysql_upgrade_info_file "$datadir")
+  echo "${version}" > "${upgrade_info_file}"
+  log_info "Storing version '${version}' information into the data dir '${upgrade_info_file}'"
+}

10.1/test/run

@@ -62,7 +62,7 @@ function test_connection() {
   local ip
   ip=$(get_container_ip $name)
   echo "  Testing MySQL connection to $ip..."
-  local max_attempts=20
+  local max_attempts=8
   local sleep_time=2
   local i
   for i in $(seq $max_attempts); do
@@ -412,6 +412,90 @@ run_doc_test() {
   echo
 }
 
+function run_upgrade_test() {
+  local tmpdir=$(mktemp -d)
+  echo "  Testing upgrade of the container image"
+  mkdir "${tmpdir}/data" && chmod -R a+rwx "${tmpdir}"
+
+  # Create MySQL container with persistent volume and set the version from too old version
+  create_container "testupg1" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \
+    -e MYSQL_DATABASE=db -v ${tmpdir}:/var/lib/mysql/data:Z
+  test_connection testupg1 user foo
+  docker stop $(get_cid testupg1) >/dev/null
+
+  # Create version file that is too old
+  echo "5.0.12" >${tmpdir}/mysql_upgrade_info
+
+  # Create another container with same data and upgrade set to 'auto'
+  create_container "testupg2" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \
+    -e MYSQL_DATABASE=db -v ${tmpdir}:/var/lib/mysql/data:Z -e MYSQL_UPGRADE=auto
+  test_connection testupg2 user foo
+  docker stop $(get_cid testupg2) >/dev/null
+
+  # Check whether some upgrade happened
+  if docker logs $(get_cid testupg2) | grep 'Checking and upgrading mysql database' &>/dev/null ; then
+    echo "Upgrade happened but it should not when upgrading from too old version"
+    return 1
+  fi
+
+  # Create version file that we can upgrade from
+  echo "10.0.12" >${tmpdir}/mysql_upgrade_info
+
+  # Create another container with same data and upgrade set to 'auto'
+  create_container "testupg3" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \
+    -e MYSQL_DATABASE=db -v ${tmpdir}:/var/lib/mysql/data:Z -e MYSQL_UPGRADE=auto
+  test_connection testupg3 user foo
+  docker stop $(get_cid testupg3) >/dev/null
+
+  # Check whether some upgrade happened
+  if ! docker logs $(get_cid testupg3) | grep 'Checking and upgrading mysql database' &>/dev/null ; then
+    echo "Upgrade did not happen but it should when upgrading from previous version"
+    return 1
+  fi
+
+  # Create version file that we don't need to upgrade from
+  echo "10.1.12" >${tmpdir}/mysql_upgrade_info
+
+  # Create another container with same data and upgrade set to 'auto'
+  create_container "testupg4" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \
+    -e MYSQL_DATABASE=db -v ${tmpdir}:/var/lib/mysql/data:Z -e MYSQL_UPGRADE=auto
+  test_connection testupg4 user foo
+  docker stop $(get_cid testupg4) >/dev/null
+
+  # Check whether some upgrade happened
+  if docker logs $(get_cid testupg4) | grep 'Checking and upgrading mysql database' &>/dev/null ; then
+    echo "Upgrade happened but it should not when upgrading from current version"
+    return 1
+  fi
+
+  # Create second container with same data and upgrade set to 'analyze'
+  create_container "testupg5" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \
+    -e MYSQL_DATABASE=db -v ${tmpdir}:/var/lib/mysql/data:Z -e MYSQL_UPGRADE=analyze
+  test_connection testupg5 user foo
+  docker stop $(get_cid testupg5) >/dev/null
+
+  # Check whether analyze happened
+  if ! docker logs $(get_cid testupg5) | grep -e '--analyze --all-databases' &>/dev/null ; then
+    echo "Analyze did not happen but it should"
+    return 1
+  fi
+
+  # Create another container with same data and upgrade set to 'optimize'
+  create_container "testupg6" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \
+    -e MYSQL_DATABASE=db -v ${tmpdir}:/var/lib/mysql/data:Z -e MYSQL_UPGRADE=optimize
+  test_connection testupg6 user foo
+  docker stop $(get_cid testupg6) >/dev/null
+
+  # Check whether optimize happened
+  if ! docker logs $(get_cid testupg6) | grep -e '--optimize --all-databases' &>/dev/null ; then
+    echo "Optimize did not happen but it should"
+    return 1
+  fi
+
+  echo "  Success!"
+  echo
+}
+
 # Tests.
 
 run_container_creation_tests
@@ -434,4 +518,8 @@ run_change_password_test
 # Replication tests
 run_replication_test
 
+# Documentation tests
 run_doc_test
+
+# Upgrade, optimaze and analyze tests
+run_upgrade_test