Enhance galera to interact over multiple clusters

This change adds a new parameter "remote_node_map"
to the Galera resource agent which allows it to consider
galera node names that are in other clusters as part of its
Galera quorum.  To achieve this, it launches pcs commands
over SSH to the remote clusters in order to view and modify
remote state variables.

WIP.

heartbeat/galera

@@ -25,7 +25,7 @@
 
 ##
 # README.
-# 
+#
 # This agent only supports being configured as a multistate Master
 # resource.
 #
@@ -49,15 +49,15 @@
 # pcs resource create db galera enable_creation=true \
 # wsrep_cluster_address="gcomm://rhel7-auto1,rhel7-auto2,rhel7-auto3" meta master-max=3 --master
 #
-# By setting the 'enable_creation' option, the database will be automatically 
+# By setting the 'enable_creation' option, the database will be automatically
 # generated at startup. The meta attribute 'master-max=3' means that all 3
 # nodes listed in the wsrep_cluster_address list will be allowed to connect
 # to the galera cluster and perform replication.
 #
 # NOTE: If you have more nodes in the pacemaker cluster then you wish
 # to have in the galera cluster, make sure to use location contraints to prevent
 # pacemaker from attempting to place a galera instance on a node that is
-# not in the 'wsrep_cluster_address" list. 
+# not in the 'wsrep_cluster_address" list.
 #
 ##
 
@@ -68,6 +68,9 @@
 . ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
 . ${OCF_FUNCTIONS_DIR}/mysql-common.sh
 
+
+SSH_CMD="${SSH} -oStrictHostKeyChecking=no"
+
 NODENAME=$(ocf_attribute_target)
 
 # It is common for some galera instances to store
@@ -226,11 +229,26 @@ pcmk1:node.1.galera;pcmk2:node.2.galera;pcmk3:node.3.galera
 
 where the galera resource started on node pcmk1 would be named
 node.1.galera in the wsrep_cluster_address
+
 </longdesc>
 <shortdesc lang="en">Pacemaker to Galera name mapping</shortdesc>
 <content type="string" default=""/>
 </parameter>
 
+<parameter name="remote_node_map" unique="0" required="0">
+<longdesc lang="en">
+A mapping of pacemaker node names to remote host SSH expressions.
+
+Allows pacemaker nodes in remote pacemaker clusters to be part of this
+Galera cluster:
+
+pcmk1:[email protected];pcmk2:[email protected]
+
+</longdesc>
+<shortdesc lang="en">Pacemaker to remote cluster nodes</shortdesc>
+<content type="string" default=""/>
+</parameter>
+
 <parameter name="check_user" unique="0" required="0">
 <longdesc lang="en">
 Cluster check user.
@@ -283,7 +301,7 @@ set_bootstrap_node()
 {
     local node=$(ocf_attribute_target $1)
 
-    ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-bootstrap" -v "true"
+    remote_crm_attribute $node -l reboot --name "${INSTANCE_ATTR_NAME}-bootstrap" -v "true"
 }
 
 clear_bootstrap_node()
@@ -310,7 +328,7 @@ clear_no_grastate()
 is_no_grastate()
 {
     local node=$(ocf_attribute_target $1)
-    ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-no-grastate" --quiet 2>/dev/null
+    remote_crm_attribute $node -l reboot --name "${INSTANCE_ATTR_NAME}-no-grastate" --quiet 2>/dev/null
 }
 
 clear_last_commit()
@@ -329,8 +347,8 @@ get_last_commit()
 
     if [ -z "$node" ]; then
        ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" --quiet 2>/dev/null
-    else 
-       ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" --quiet 2>/dev/null
+    else
+       remote_crm_attribute $node -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" --quiet 2>/dev/null
     fi
 }
 
@@ -351,7 +369,7 @@ get_safe_to_bootstrap()
     if [ -z "$node" ]; then
         ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-safe-to-bootstrap" --quiet 2>/dev/null
     else
-        ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-safe-to-bootstrap" --quiet 2>/dev/null
+        remote_crm_attribute $node -l reboot --name "${INSTANCE_ATTR_NAME}-safe-to-bootstrap" --quiet 2>/dev/null
     fi
 }
 
@@ -410,16 +428,34 @@ master_exists()
     fi
     # determine if a master instance is already up and is healthy
     crm_mon --as-xml | grep "resource.*id=\"${INSTANCE_ATTR_NAME}\".*role=\"Master\".*active=\"true\".*orphaned=\"false\".*failed=\"false\"" > /dev/null 2>&1
-    return $?
+
+    local master_exists_local=$?
+
+    if [ $master_exists_local -eq 0 ]; then
+        ocf_log info "Detected that a master exists for the local cluster"
+    fi
+
+    # if not, and we have remote nodes, check those also
+    if [ $master_exists_local -ne 0 ] && [ -n "$OCF_RESKEY_remote_node_map" ]; then
+        for remote_ssh in $(echo "$OCF_RESKEY_remote_node_map" | tr ';' '\n' | tr -d ' ' | sed 's/:/ /' | awk -F' ' '{print $2;}' | sort | uniq); do
+            $SSH_CMD $remote_ssh crm_mon --as-xml | grep "resource.*id=\"${INSTANCE_ATTR_NAME}\".*role=\"Master\".*active=\"true\".*orphaned=\"false\".*failed=\"false\"" > /dev/null 2>&1
+            if [ $? -eq 0 ]; then
+                ocf_log info "Detected that a master exists for the remote cluster $remote_ssh"
+                return $?
+            fi
+        done
+    fi
+
+    return $master_exists_local
 }
 
 clear_master_score()
 {
     local node=$(ocf_attribute_target $1)
     if [ -z "$node" ]; then
         $CRM_MASTER -D
-    else 
-        $CRM_MASTER -D -N $node
+    else
+        remote_crm_master $node -D
     fi
 }
 
@@ -429,8 +465,51 @@ set_master_score()
 
     if [ -z "$node" ]; then
         $CRM_MASTER -v 100
-    else 
-        $CRM_MASTER -N $node -v 100
+    else
+        remote_crm_master $node -v 100
+    fi
+}
+
+get_remote_node()
+{
+    local node=$1
+    if [ -z "$OCF_RESKEY_remote_node_map" ]; then
+        return
+    else
+        local retval=$(echo "$OCF_RESKEY_remote_node_map" | tr ';' '\n' | tr -d ' ' | sed 's/:/ /' | awk -F' ' '$1=="'"$node"'" {print $2;exit}')
+        if [ -z "$retval" ]; then
+            return
+        else
+            echo $retval
+        fi
+    fi
+}
+
+remote_crm_master()
+{
+    local node=$1
+    shift
+
+    local remote_ssh=$(get_remote_node $node)
+
+    if [ -z "$remote_ssh" ]; then
+        $CRM_MASTER -N $node "$@"
+    else
+        $SSH_CMD $remote_ssh $CRM_MASTER -r ${INSTANCE_ATTR_NAME} -N $node "$@"
+    fi
+}
+
+remote_crm_attribute()
+{
+    local node=$1
+    shift
+
+    local remote_ssh=$(get_remote_node $node)
+
+    if [ -z "$remote_ssh" ]; then
+        ${HA_SBIN_DIR}/crm_attribute -N $node "$@"
+    else
+        $SSH_CMD $remote_ssh ${HA_SBIN_DIR}/crm_attribute -N $node "$@"
     fi
 }