support multisite in ceph2 setup

README.md

@@ -308,14 +308,14 @@ docker-compose exec ceph /docker/sso/sso-disable.sh
 RGW_MULTISITE=1
 ```
 
-* Start ceph (cluster 1) + ceph cluster 2:
+* Start ceph (cluster 1) + ceph2 (cluster 2):
 ```
-docker-compose up -d --scale ceph-cluster2=1
+docker-compose up -d --scale ceph2=1 --scale prometheus2=1
 ```
 
 * Run 100s duration [benchmark](https://github.com/markhpc/hsbench#usage):
 ```
-docker-compose exec ceph-cluster2 bash
+docker-compose exec ceph2 bash
 hsbench -a <rgw-user-access-key> -s <rgw-user-secret-key> -u http://127.0.0.1:8000 -z 4K -d 100 -t 10 -b 10
 ```
 

docker-compose.yml

@@ -53,12 +53,6 @@ services:
         ports: ['${CEPH_PROXY_HOST_PORT:-4200}:4200','${CEPH_HOST_PORT:-11000}:11000']
         scale: 1
 
-    ceph-cluster2:
-        extends:
-            service: ceph-base
-        container_name: ceph-cluster2
-        hostname: ceph-cluster2
-
     ceph-host2:
         extends:
             service: ceph-base

docker/ceph/set-rgw.sh

@@ -8,7 +8,7 @@
 readonly RGW_REALM_USER_ACCESS_KEY_FILE="/tmp/rgw-user-access-key.txt"
 readonly RGW_REALM_USER_SECRET_KEY=('xSZUdYky0bTctAdCEEW8ikhfBVKsBV5LFYL82vvh' 'GsOwxjOfxAwM2It2mBiB5A5CJFxu73GjpNjCROvx')
 readonly RGW_REALM_USER_SECRET_KEY_FILE="/tmp/rgw-user-secret-key.txt"
 readonly RGW_REALM_USER_NUM=1
 RGW_DAEMON_PORT=8000
 
 truncate -s0 ${RGW_REALM_USER_ACCESS_KEY_FILE}
@@ -30,11 +30,11 @@
     "$CEPH_BIN"/radosgw-admin user create --uid "${USER_UID}" \
         --display-name "${RGW_REALM_USER_NAME}${USER_NUM}" \
         --access-key "${RGW_REALM_USER_ACCESS_KEY[${SECRET_INDEX}]}" --secret "${RGW_REALM_USER_SECRET_KEY[${SECRET_INDEX}]}" \
         ${NAMESPACE_OPTIONS} --system
 
     # Create MFA TOTP token.
     "$CEPH_BIN"/radosgw-admin mfa create --uid="${USER_UID}" --totp-serial=1 --totp-seed=23456723 --totp-seed-type=base32 \
         ${NAMESPACE_OPTIONS}
 }
 
 add_placement_targets_and_storage_classes() {
@@ -74,7 +74,7 @@
 
     rm -f "$RGW_DAEMON_PID_FILE"
 
     if [[ $(grep "\[$RGW_DAEMON_NAME\]" "$CEPH_CONF" | grep -v grep | wc -l) == 0 ]]; then
         "$CEPH_BIN"/ceph -c "$CEPH_CONF" auth get-or-create "$RGW_DAEMON_NAME" \
             mon 'allow rw' osd 'allow rwx' mgr 'allow rw' \
             >> "$CEPH_CONF_PATH"/keyring
@@ -146,7 +146,7 @@
         echo -n "{$(cat ${RGW_REALM_USER_ACCESS_KEY_FILE} | sed 's/,$//')}" > "${RGW_REALM_USER_ACCESS_KEY_FILE}"
         echo -n "{$(cat ${RGW_REALM_USER_SECRET_KEY_FILE} | sed 's/,$//')}" > "${RGW_REALM_USER_SECRET_KEY_FILE}"
     else
-        readonly FIRST_CLUSTER_HOSTNAME=$(hostname | sed -e 's/-cluster.//')
+        readonly FIRST_CLUSTER_HOSTNAME=$(hostname | sed -e 's/2//g')
         readonly CLUSTER2_PULL_REALM="${REALM_NAME_PREFIX}1"
         readonly CLUSTER2_ZONEGROUP="${ZONEGROUP_NAME_PREFIX}1-${CLUSTER2_PULL_REALM}"
         readonly CLUSTER2_ZONE="${ZONE_NAME_PREFIX}3-${CLUSTER2_ZONEGROUP}"

docker/ceph/set-start-env.sh

@@ -7,21 +7,14 @@
 [[ -d "$MGR_PYTHON_PATH"/dashboard/frontend ]] && export IS_UPSTREAM_LUMINOUS=0
 [[ -z "$RGW" ]] && export RGW=1
 
-export IS_FIRST_CLUSTER=$(hostname | grep -v cluster | wc -l)
+export IS_FIRST_CLUSTER=$(hostname | grep -v ceph2 | wc -l)
 
-[[ ("$IS_CEPH_RPM" == 0 || -n "$CEPH_REPO_DIR") && "$IS_UPSTREAM_LUMINOUS" == 0 && "$IS_FIRST_CLUSTER" == 1 ]] \
+[[ ("$IS_CEPH_RPM" == 0 || -n "$CEPH_REPO_DIR") && "$IS_UPSTREAM_LUMINOUS" == 0 ]] \
     && export FRONTEND_BUILD_REQUIRED=1
 FRONTEND_BUILD_OPTIONS=${FRONTEND_BUILD_OPTIONS:-"--deleteOutputPath=false"}
 
 if [[ "$RGW_MULTISITE" == 1 ]]; then
     export RGW=0  # Required to prevent vstart from starting any rgw daemon.
-
-    if [[ "$IS_FIRST_CLUSTER" == 0 ]]; then
-        export FS=0
-        export MDS=0
-        export MGR=0
-        export MON=1
-    fi
 fi
 
 RGW_DEBUG=''
@@ -38,4 +31,4 @@
     HTTP_PROTO='https'
 fi
 export HTTP_PROTO
 export CEPH_MGR_DASHBOARD_PORT=$(($CEPH_PORT + 1000))

docker/ceph/start-ceph.sh

@@ -2,7 +2,7 @@
 
 set -e
 
 source /docker/set-start-env.sh
 
 # Build frontend:
 if [[ "$FRONTEND_BUILD_REQUIRED" == 1 ]]; then
@@ -11,7 +11,7 @@
     # Set dev server proxy:
     TARGET_URL="${HTTP_PROTO}://${HOSTNAME}:${CEPH_MGR_DASHBOARD_PORT}"
     [[ -n "${DASHBOARD_URL}" ]] && TARGET_URL=${DASHBOARD_URL}
     jq "(.[] | .target)=\""${TARGET_URL}"\"" proxy.conf.json.sample > proxy.conf.json
 
     if [[ "$CEPH_VERSION" == '13' ]]; then
         rm -rf package-lock.json node_modules/@angular/cli
@@ -45,10 +45,10 @@
     exit 0
 fi
 
 rm -rf "$CEPH_CONF_PATH"/*
 
 cd /ceph/build
 ../src/vstart.sh ${VSTART_OPTIONS}
 
 echo 'vstart.sh completed!'
 
@@ -58,10 +58,8 @@
 fi
 
 # Enable prometheus module
-if [[ "$IS_FIRST_CLUSTER" == 1 ]]; then
-    "$CEPH_BIN"/ceph mgr module enable prometheus
-    echo 'Prometheus mgr module enabled.'
-fi
+"$CEPH_BIN"/ceph mgr module enable prometheus
+echo 'Prometheus mgr module enabled.'
 
 #Enable and set test_orchestrator module
 if [[ "$TEST_ORCHESTRATOR" == 1 ]]; then
@@ -75,7 +73,7 @@
 
 # Disable ssl (if selected)
 readonly VSTART_HAS_SSL_FLAG=$(cat /ceph/src/vstart.sh | grep DASHBOARD_SSL | wc -l)
-if [[ "$DASHBOARD_SSL" == 0 && "$VSTART_HAS_SSL_FLAG" == 0 && "$IS_FIRST_CLUSTER" == 1 ]]; then
+if [[ "$DASHBOARD_SSL" == 0 && "$VSTART_HAS_SSL_FLAG" == 0 ]]; then
     echo "Disabling SSL..."
 
     SSL_OPTIONS='--force'
@@ -91,7 +89,7 @@
 fi
 
 # Secondary cluster start (or upstream mimic start) ends here.
-[[ "$IS_FIRST_CLUSTER" == 0 || "$CEPH_VERSION" -le '13' ]] && exit 0
+[[ "$CEPH_VERSION" -le '13' ]] && exit 0
 
 # Create dashboard "test" user:
 [[ "$CEPH_VERSION" -gt '14' ]] && DASHBOARD_USER_CREATE_OPTIONS='--force-password'