From f9d030a15f1576dd3392e71470a982bfebc14ef5 Mon Sep 17 00:00:00 2001 From: N Balachandran Date: Fri, 23 Aug 2024 17:39:16 +0530 Subject: [PATCH] rbd-mirror: allow mirroring to a different namespace Allows a namespace in a pool to be mirrored to a differently named namespace in the secondary cluster. Signed-off-by: N Balachandran --- doc/man/8/rbd.rst | 5 +- qa/workunits/rbd/rbd_mirror.sh | 133 +++++++++++------- qa/workunits/rbd/rbd_mirror_bootstrap.sh | 6 +- qa/workunits/rbd/rbd_mirror_ha.sh | 2 +- qa/workunits/rbd/rbd_mirror_helpers.sh | 50 ++++--- qa/workunits/rbd/rbd_mirror_stress.sh | 6 +- src/cls/rbd/cls_rbd.cc | 56 ++++++++ src/cls/rbd/cls_rbd_client.cc | 48 +++++++ src/cls/rbd/cls_rbd_client.h | 8 ++ src/include/rbd/librbd.h | 5 + src/include/rbd/librbd.hpp | 4 + src/librbd/api/Mirror.cc | 45 +++++- src/librbd/api/Mirror.h | 10 ++ src/librbd/librbd.cc | 50 +++++++ src/pybind/rbd/c_rbd.pxd | 6 + src/pybind/rbd/mock_rbd.pxi | 8 ++ src/pybind/rbd/rbd.pyx | 46 ++++++ src/test/cli/rbd/help.t | 24 ++-- src/test/cls_rbd/test_cls_rbd.cc | 8 ++ src/test/pybind/test_rbd.py | 16 +++ .../rbd_mirror/test_mock_NamespaceReplayer.cc | 12 +- src/test/rbd_mirror/test_mock_PoolReplayer.cc | 22 ++- src/tools/rbd/ArgumentTypes.h | 1 + src/tools/rbd/action/MirrorPool.cc | 73 ++++++++-- src/tools/rbd_mirror/NamespaceReplayer.cc | 71 +++++++++- src/tools/rbd_mirror/NamespaceReplayer.h | 16 ++- src/tools/rbd_mirror/PoolReplayer.cc | 45 ++++-- src/tools/rbd_mirror/PoolReplayer.h | 3 +- 28 files changed, 641 insertions(+), 138 deletions(-) diff --git a/doc/man/8/rbd.rst b/doc/man/8/rbd.rst index 0f490506aa6b..3da2930d2cc1 100644 --- a/doc/man/8/rbd.rst +++ b/doc/man/8/rbd.rst @@ -543,8 +543,9 @@ Commands :command:`mirror pool info` [*pool-name*] Show information about the pool or namespace mirroring configuration. - For a pool, it includes mirroring mode, peer UUID, remote cluster name, - and remote client name. For a namespace, it includes only mirroring mode. + For both pools and namespaces, it includes the mirroring mode, mirror uuid + and remote namespace. For pools, it additonally includes the site name, + peer uuid, remote cluster name, and remote client name. :command:`mirror pool peer add` [*pool-name*] *remote-cluster-spec* Add a mirroring peer to a pool. diff --git a/qa/workunits/rbd/rbd_mirror.sh b/qa/workunits/rbd/rbd_mirror.sh index 1cda355039eb..3229972828a7 100755 --- a/qa/workunits/rbd/rbd_mirror.sh +++ b/qa/workunits/rbd/rbd_mirror.sh @@ -37,12 +37,12 @@ set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1" set_image_meta ${CLUSTER2} ${POOL} ${image} "key2" "value2" wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} write_image ${CLUSTER2} ${POOL} ${image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown' fi -compare_images ${POOL} ${image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} compare_image_meta ${CLUSTER1} ${POOL} ${image} "key1" "value1" compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2" @@ -53,19 +53,19 @@ create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image1} ${RBD_MIRROR_MODE} write_image ${CLUSTER2} ${POOL} ${image1} 100 start_mirrors ${CLUSTER1} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image1} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown' fi -compare_images ${POOL} ${image1} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image1} testlog "TEST: test the first image is replaying after restart" write_image ${CLUSTER2} ${POOL} ${image} 100 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} -compare_images ${POOL} ${image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then testlog "TEST: stop/start/restart mirror via admin socket" @@ -173,7 +173,7 @@ wait_for_image_in_omap ${CLUSTER2} ${POOL} create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${RBD_MIRROR_MODE} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} write_image ${CLUSTER2} ${POOL} ${image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' testlog "TEST: failover and failback" @@ -187,10 +187,10 @@ wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' promote_image ${CLUSTER2} ${POOL} ${image} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} write_image ${CLUSTER2} ${POOL} ${image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} -compare_images ${POOL} ${image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} # failover (unmodified) demote_image ${CLUSTER2} ${POOL} ${image} @@ -207,10 +207,10 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' promote_image ${CLUSTER2} ${POOL} ${image} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' -compare_images ${POOL} ${image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} # failover demote_image ${CLUSTER2} ${POOL} ${image} @@ -220,10 +220,10 @@ wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' promote_image ${CLUSTER1} ${POOL} ${image} wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image} write_image ${CLUSTER1} ${POOL} ${image} 100 -wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' wait_for_replaying_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} -compare_images ${POOL} ${image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} # failback demote_image ${CLUSTER1} ${POOL} ${image} @@ -233,10 +233,10 @@ wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' promote_image ${CLUSTER2} ${POOL} ${image} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} write_image ${CLUSTER2} ${POOL} ${image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' -compare_images ${POOL} ${image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} testlog "TEST: failover / failback loop" for i in `seq 1 20`; do @@ -246,7 +246,7 @@ for i in `seq 1 20`; do wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' promote_image ${CLUSTER1} ${POOL} ${image} wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image} - wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' demote_image ${CLUSTER1} ${POOL} ${image} @@ -255,7 +255,7 @@ for i in `seq 1 20`; do wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' promote_image ${CLUSTER2} ${POOL} ${image} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' done @@ -271,7 +271,7 @@ create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${force_promote_image} ${RBD_ write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${force_promote_image} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped' promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force' @@ -302,14 +302,14 @@ else enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${RBD_MIRROR_MODE} fi wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${PARENT_POOL} ${parent_image} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} -compare_images ${PARENT_POOL} ${parent_image} +compare_images ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${PARENT_POOL} ${parent_image} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${clone_image} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} -compare_images ${POOL} ${clone_image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${clone_image} remove_image_retry ${CLUSTER2} ${POOL} ${clone_image} testlog " - clone v1" @@ -383,11 +383,11 @@ create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap1' write_image ${CLUSTER2} ${POOL} ${dp_image} 100 create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap2' write_image ${CLUSTER2} ${POOL} ${dp_image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${dp_image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${dp_image} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} -compare_images ${POOL} ${dp_image}@snap1 -compare_images ${POOL} ${dp_image}@snap2 -compare_images ${POOL} ${dp_image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${dp_image}@snap1 +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${dp_image}@snap2 +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${dp_image} remove_image_retry ${CLUSTER2} ${POOL} ${dp_image} testlog "TEST: disable mirroring / delete non-primary image" @@ -436,8 +436,8 @@ if [ "${RBD_MIRROR_MODE}" = "journal" ]; then wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present' wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${i} - wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${i} - compare_images ${POOL} ${i} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${i} + compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${i} done testlog "TEST: remove mirroring pool" @@ -454,9 +454,9 @@ if [ "${RBD_MIRROR_MODE}" = "journal" ]; then create_image ${CLUSTER2} ${POOL} ${rdp_image} 128 --data-pool ${pool} write_image ${CLUSTER2} ${pool} ${image} 100 write_image ${CLUSTER2} ${POOL} ${rdp_image} 100 - wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${pool} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${pool} ${pool} ${image} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${pool} ${image} - wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${rdp_image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${rdp_image} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${rdp_image} for cluster in ${CLUSTER1} ${CLUSTER2}; do CEPH_ARGS='' ceph --cluster ${cluster} osd pool rm ${pool} ${pool} --yes-i-really-really-mean-it @@ -519,12 +519,12 @@ wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${image} wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS2} ${image} write_image ${CLUSTER2} ${POOL}/${NS1} ${image} 100 write_image ${CLUSTER2} ${POOL}/${NS2} ${image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${POOL}/${NS1} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${POOL}/${NS2} ${image} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${image} -compare_images ${POOL}/${NS1} ${image} -compare_images ${POOL}/${NS2} ${image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${POOL}/${NS1} ${image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${POOL}/${NS2} ${image} testlog " - disable mirroring / delete image" remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${image} @@ -533,6 +533,43 @@ wait_for_image_present ${CLUSTER1} ${POOL}/${NS1} ${image} 'deleted' wait_for_image_present ${CLUSTER1} ${POOL}/${NS2} ${image} 'deleted' remove_image_retry ${CLUSTER2} ${POOL}/${NS2} ${image} + +testlog "TEST: mirror to a different remote namespace" +testlog " - replay" +NS3=ns3 +NS4=ns4 +rbd --cluster ${CLUSTER1} namespace create ${POOL}/${NS3} +rbd --cluster ${CLUSTER2} namespace create ${POOL}/${NS4} +rbd --cluster ${CLUSTER1} mirror pool enable ${POOL}/${NS3} ${MIRROR_POOL_MODE} --remote-namespace ${NS4} +rbd --cluster ${CLUSTER2} mirror pool enable ${POOL}/${NS4} ${MIRROR_POOL_MODE} --remote-namespace ${NS3} +create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS4} ${image} ${RBD_MIRROR_MODE} +wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS3} ${image} +write_image ${CLUSTER2} ${POOL}/${NS4} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS3} ${POOL}/${NS4} ${image} +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS3} ${image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS3} ${POOL}/${NS4} ${image} #FIX THIS! + + +testlog " - disable mirroring and re-enable without remote-namespace" +remove_image_retry ${CLUSTER2} ${POOL}/${NS4} ${image} +wait_for_image_present ${CLUSTER1} ${POOL}/${NS3} ${image} 'deleted' +rbd --cluster ${CLUSTER1} mirror pool disable ${POOL}/${NS3} +rbd --cluster ${CLUSTER2} mirror pool disable ${POOL}/${NS4} +rbd --cluster ${CLUSTER2} namespace create ${POOL}/${NS3} +rbd --cluster ${CLUSTER2} mirror pool enable ${POOL}/${NS3} ${MIRROR_POOL_MODE} +rbd --cluster ${CLUSTER1} mirror pool enable ${POOL}/${NS3} ${MIRROR_POOL_MODE} +create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS3} ${image} ${RBD_MIRROR_MODE} +wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS3} ${image} +write_image ${CLUSTER2} ${POOL}/${NS3} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS3} ${POOL}/${NS3} ${image} +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS3} ${image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS3} ${POOL}/${NS3} ${image} +remove_image_retry ${CLUSTER2} ${POOL}/${NS3} ${image} +wait_for_image_present ${CLUSTER1} ${POOL}/${NS3} ${image} 'deleted' +rbd --cluster ${CLUSTER1} mirror pool disable ${POOL}/${NS3} +rbd --cluster ${CLUSTER2} mirror pool disable ${POOL}/${NS3} + + testlog " - data pool" dp_image=test_data_pool create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS1} ${dp_image} ${RBD_MIRROR_MODE} 128 --data-pool ${PARENT_POOL} @@ -542,9 +579,9 @@ wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${dp_image} data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL}/${NS1} ${dp_image}) test "${data_pool}" = "${PARENT_POOL}" write_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${dp_image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${POOL}/${NS1} ${dp_image} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} -compare_images ${POOL}/${NS1} ${dp_image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${POOL}/${NS1} ${dp_image} remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${dp_image} testlog "TEST: simple image resync" @@ -553,7 +590,7 @@ wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} -compare_images ${POOL} ${image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then testlog "TEST: image resync while replayer is stopped" @@ -566,7 +603,7 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} - compare_images ${POOL} ${image} + compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} fi testlog "TEST: request image resync while daemon is offline" @@ -577,7 +614,7 @@ wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} -compare_images ${POOL} ${image} +compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} remove_image_retry ${CLUSTER2} ${POOL} ${image} if [ "${RBD_MIRROR_MODE}" = "journal" ]; then @@ -588,7 +625,7 @@ if [ "${RBD_MIRROR_MODE}" = "journal" ]; then testlog " - replay stopped after disconnect" wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" disconnect_image ${CLUSTER2} ${POOL} ${image} test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" @@ -600,9 +637,9 @@ if [ "${RBD_MIRROR_MODE}" = "journal" ]; then wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" - compare_images ${POOL} ${image} + compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} testlog " - disconnected after max_concurrent_object_sets reached" if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then @@ -628,25 +665,25 @@ if [ "${RBD_MIRROR_MODE}" = "journal" ]; then wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" - compare_images ${POOL} ${image} + compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} testlog " - rbd_mirroring_resync_after_disconnect config option" set_image_meta ${CLUSTER2} ${POOL} ${image} \ conf_rbd_mirroring_resync_after_disconnect true - wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} image_id=$(get_image_id ${CLUSTER1} ${POOL} ${image}) disconnect_image ${CLUSTER2} ${POOL} ${image} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" - compare_images ${POOL} ${image} + compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} set_image_meta ${CLUSTER2} ${POOL} ${image} \ conf_rbd_mirroring_resync_after_disconnect false - wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} disconnect_image ${CLUSTER2} ${POOL} ${image} test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} diff --git a/qa/workunits/rbd/rbd_mirror_bootstrap.sh b/qa/workunits/rbd/rbd_mirror_bootstrap.sh index 412e84c88a64..3ddb0aa219b7 100755 --- a/qa/workunits/rbd/rbd_mirror_bootstrap.sh +++ b/qa/workunits/rbd/rbd_mirror_bootstrap.sh @@ -38,7 +38,7 @@ create_image_and_enable_mirror ${CLUSTER1} ${POOL} image1 wait_for_image_replay_started ${CLUSTER2} ${POOL} image1 write_image ${CLUSTER1} ${POOL} image1 100 -wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} image1 +wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${POOL} image1 wait_for_replaying_status_in_pool_dir ${CLUSTER2} ${POOL} image1 testlog "TEST: verify rx-tx direction" @@ -54,12 +54,12 @@ enable_mirror ${CLUSTER2} ${PARENT_POOL} image2 wait_for_image_replay_started ${CLUSTER2} ${PARENT_POOL} image1 write_image ${CLUSTER1} ${PARENT_POOL} image1 100 -wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${PARENT_POOL} image1 +wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${PARENT_POOL} ${PARENT_POOL} image1 wait_for_replaying_status_in_pool_dir ${CLUSTER2} ${PARENT_POOL} image1 wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} image2 write_image ${CLUSTER2} ${PARENT_POOL} image2 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} image2 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${PARENT_POOL} image2 wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} image2 testlog "TEST: pool replayer and callout cleanup when peer is updated" diff --git a/qa/workunits/rbd/rbd_mirror_ha.sh b/qa/workunits/rbd/rbd_mirror_ha.sh index 1e43712a6315..1301657a7022 100755 --- a/qa/workunits/rbd/rbd_mirror_ha.sh +++ b/qa/workunits/rbd/rbd_mirror_ha.sh @@ -79,7 +79,7 @@ test_replay() wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} \ 'down+unknown' fi - compare_images ${POOL} ${image} + compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} done } diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh index 1c3062891768..1b1436db74d7 100755 --- a/qa/workunits/rbd/rbd_mirror_helpers.sh +++ b/qa/workunits/rbd/rbd_mirror_helpers.sh @@ -743,17 +743,18 @@ wait_for_journal_replay_complete() { local local_cluster=$1 local cluster=$2 - local pool=$3 - local image=$4 + local local_pool=$3 + local remote_pool=$4 + local image=$5 local s master_pos mirror_pos last_mirror_pos local master_tag master_entry mirror_tag mirror_entry while true; do for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do sleep ${s} - flush "${local_cluster}" "${pool}" "${image}" - master_pos=$(get_master_journal_position "${cluster}" "${pool}" "${image}") - mirror_pos=$(get_mirror_journal_position "${cluster}" "${pool}" "${image}") + flush "${local_cluster}" "${local_pool}" "${image}" + master_pos=$(get_master_journal_position "${cluster}" "${remote_pool}" "${image}") + mirror_pos=$(get_mirror_journal_position "${cluster}" "${remote_pool}" "${image}") test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0 test "${mirror_pos}" != "${last_mirror_pos}" && break done @@ -796,21 +797,22 @@ wait_for_snapshot_sync_complete() { local local_cluster=$1 local cluster=$2 - local pool=$3 - local image=$4 + local local_pool=$3 + local remote_pool=$4 + local image=$5 - local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}-${image}.status) - local local_status_log=${TEMPDIR}/$(mkfname ${local_cluster}-${pool}-${image}.status) + local status_log=${TEMPDIR}/$(mkfname ${cluster}-${remote_pool}-${image}.status) + local local_status_log=${TEMPDIR}/$(mkfname ${local_cluster}-${local_pool}-${image}.status) - mirror_image_snapshot "${cluster}" "${pool}" "${image}" - get_newest_mirror_snapshot "${cluster}" "${pool}" "${image}" "${status_log}" + mirror_image_snapshot "${cluster}" "${remote_pool}" "${image}" + get_newest_mirror_snapshot "${cluster}" "${remote_pool}" "${image}" "${status_log}" local snapshot_id=$(xmlstarlet sel -t -v "//snapshot/id" < ${status_log}) while true; do for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do sleep ${s} - get_newest_mirror_snapshot "${local_cluster}" "${pool}" "${image}" "${local_status_log}" + get_newest_mirror_snapshot "${local_cluster}" "${local_pool}" "${image}" "${local_status_log}" local primary_snapshot_id=$(xmlstarlet sel -t -v "//snapshot/namespace/primary_snap_id" < ${local_status_log}) test "${snapshot_id}" = "${primary_snapshot_id}" && return 0 @@ -825,13 +827,14 @@ wait_for_replay_complete() { local local_cluster=$1 local cluster=$2 - local pool=$3 - local image=$4 + local local_pool=$3 + local remote_pool=$4 + local image=$5 if [ "${RBD_MIRROR_MODE}" = "journal" ]; then - wait_for_journal_replay_complete ${local_cluster} ${cluster} ${pool} ${image} + wait_for_journal_replay_complete ${local_cluster} ${cluster} ${local_pool} ${remote_pool} ${image} elif [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then - wait_for_snapshot_sync_complete ${local_cluster} ${cluster} ${pool} ${image} + wait_for_snapshot_sync_complete ${local_cluster} ${cluster} ${local_pool} ${remote_pool} ${image} else return 1 fi @@ -1298,16 +1301,19 @@ show_diff() compare_images() { - local pool=$1 - local image=$2 local ret=0 + local local_cluster=$1 + local cluster=$2 + local local_pool=$3 + local remote_pool=$4 + local image=$5 - local rmt_export=${TEMPDIR}/$(mkfname ${CLUSTER2}-${pool}-${image}.export) - local loc_export=${TEMPDIR}/$(mkfname ${CLUSTER1}-${pool}-${image}.export) + local rmt_export=${TEMPDIR}/$(mkfname ${cluster}-${remote_pool}-${image}.export) + local loc_export=${TEMPDIR}/$(mkfname ${local_cluster}-${local_pool}-${image}.export) rm -f ${rmt_export} ${loc_export} - rbd --cluster ${CLUSTER2} export ${pool}/${image} ${rmt_export} - rbd --cluster ${CLUSTER1} export ${pool}/${image} ${loc_export} + rbd --cluster ${cluster} export ${remote_pool}/${image} ${rmt_export} + rbd --cluster ${local_cluster} export ${local_pool}/${image} ${loc_export} if ! cmp ${rmt_export} ${loc_export} then show_diff ${rmt_export} ${loc_export} diff --git a/qa/workunits/rbd/rbd_mirror_stress.sh b/qa/workunits/rbd/rbd_mirror_stress.sh index baf0c9f1a8f8..b0a85e8a48a5 100755 --- a/qa/workunits/rbd/rbd_mirror_stress.sh +++ b/qa/workunits/rbd/rbd_mirror_stress.sh @@ -111,7 +111,7 @@ do snap_name="snap${i}" create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name} if [ -n "${clean_snap_name}" ]; then @@ -124,7 +124,7 @@ do done wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${clean_snap_name} for i in `seq 1 10` @@ -173,7 +173,7 @@ do image="image_${i}" create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image} wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name} compare_image_snaps ${POOL} ${image} ${snap_name} done diff --git a/src/cls/rbd/cls_rbd.cc b/src/cls/rbd/cls_rbd.cc index b84630e14860..7643599683cf 100644 --- a/src/cls/rbd/cls_rbd.cc +++ b/src/cls/rbd/cls_rbd.cc @@ -4624,6 +4624,7 @@ static const std::string STATUS_GLOBAL_KEY_PREFIX("status_global_"); static const std::string REMOTE_STATUS_GLOBAL_KEY_PREFIX("remote_status_global_"); static const std::string INSTANCE_KEY_PREFIX("instance_"); static const std::string MIRROR_IMAGE_MAP_KEY_PREFIX("image_map_"); +static const std::string REMOTE_NAMESPACE("remote_namespace"); std::string peer_key(const std::string &uuid) { return PEER_KEY_PREFIX + uuid; @@ -5920,6 +5921,54 @@ int mirror_mode_set(cls_method_context_t hctx, bufferlist *in, if (r < 0) { return r; } + + r = remove_key(hctx, mirror::REMOTE_NAMESPACE); + if (r < 0) { + return r; + } + } + return 0; +} + +int mirror_namespace_get(cls_method_context_t hctx, bufferlist *in, + bufferlist *out) { + std::string mirror_ns_decode; + int r = read_key(hctx, mirror::REMOTE_NAMESPACE, &mirror_ns_decode); + if (r < 0) { + CLS_ERR("error getting mirror namespace: %s", cpp_strerror(r).c_str()); + return r; + } + + encode(mirror_ns_decode, *out); + return 0; +} + +int mirror_namespace_set(cls_method_context_t hctx, bufferlist *in, + bufferlist *out) { + std::string mirror_namespace; + try { + auto bl_it = in->cbegin(); + decode(mirror_namespace, bl_it); + } catch (const ceph::buffer::error &err) { + return -EINVAL; + } + + uint32_t mirror_mode; + int r = read_key(hctx, mirror::MODE, &mirror_mode); + if (r < 0 && r != -ENOENT) { + return r; + } else if (r == 0 && mirror_mode != cls::rbd::MIRROR_MODE_DISABLED) { + CLS_ERR("cannot set mirror remote namespace while mirroring enabled"); + return -EINVAL; + } + + bufferlist bl; + encode(mirror_namespace, bl); + + r = cls_cxx_map_set_val(hctx, mirror::REMOTE_NAMESPACE, &bl); + if (r < 0) { + CLS_ERR("error setting mirror namespace: %s", cpp_strerror(r).c_str()); + return r; } return 0; } @@ -8278,6 +8327,8 @@ CLS_INIT(rbd) cls_method_handle_t h_mirror_uuid_set; cls_method_handle_t h_mirror_mode_get; cls_method_handle_t h_mirror_mode_set; + cls_method_handle_t h_mirror_namespace_get; + cls_method_handle_t h_mirror_namespace_set; cls_method_handle_t h_mirror_peer_ping; cls_method_handle_t h_mirror_peer_list; cls_method_handle_t h_mirror_peer_add; @@ -8575,6 +8626,11 @@ CLS_INIT(rbd) cls_register_cxx_method(h_class, "mirror_mode_set", CLS_METHOD_RD | CLS_METHOD_WR, mirror_mode_set, &h_mirror_mode_set); + cls_register_cxx_method(h_class, "mirror_namespace_get", CLS_METHOD_RD, + mirror_namespace_get, &h_mirror_namespace_get); + cls_register_cxx_method(h_class, "mirror_namespace_set", + CLS_METHOD_RD | CLS_METHOD_WR, + mirror_namespace_set, &h_mirror_namespace_set); cls_register_cxx_method(h_class, "mirror_peer_ping", CLS_METHOD_RD | CLS_METHOD_WR, mirror_peer_ping, &h_mirror_peer_ping); diff --git a/src/cls/rbd/cls_rbd_client.cc b/src/cls/rbd/cls_rbd_client.cc index ad480c47d5c4..0161fdc46612 100644 --- a/src/cls/rbd/cls_rbd_client.cc +++ b/src/cls/rbd/cls_rbd_client.cc @@ -1882,6 +1882,54 @@ int mirror_mode_set(librados::IoCtx *ioctx, return 0; } +void mirror_remote_namespace_get_start(librados::ObjectReadOperation *op) { + bufferlist bl; + op->exec("rbd", "mirror_namespace_get", bl); +} + +int mirror_remote_namespace_get_finish(bufferlist::const_iterator *it, + std::string *mirror_namespace) { + try { + decode(*mirror_namespace, *it); + } catch (const ceph::buffer::error &err) { + return -EBADMSG; + } + return 0; +} + +int mirror_remote_namespace_get(librados::IoCtx *ioctx, + std::string *mirror_namespace) { + librados::ObjectReadOperation op; + mirror_remote_namespace_get_start(&op); + + bufferlist out_bl; + int r = ioctx->operate(RBD_MIRRORING, &op, &out_bl); + if (r < 0) { + return r; + } + + auto it = out_bl.cbegin(); + r = mirror_remote_namespace_get_finish(&it, mirror_namespace); + if (r < 0) { + return r; + } + return 0; +} + +int mirror_remote_namespace_set(librados::IoCtx *ioctx, + const std::string &mirror_namespace) { + bufferlist in_bl; + encode(mirror_namespace, in_bl); + + bufferlist out_bl; + int r = ioctx->exec(RBD_MIRRORING, "rbd", "mirror_namespace_set", in_bl, + out_bl); + if (r < 0) { + return r; + } + return 0; +} + void mirror_peer_list_start(librados::ObjectReadOperation *op) { bufferlist bl; op->exec("rbd", "mirror_peer_list", bl); diff --git a/src/cls/rbd/cls_rbd_client.h b/src/cls/rbd/cls_rbd_client.h index 4005c51836c7..c0bd5982894e 100644 --- a/src/cls/rbd/cls_rbd_client.h +++ b/src/cls/rbd/cls_rbd_client.h @@ -389,6 +389,14 @@ int mirror_mode_get(librados::IoCtx *ioctx, int mirror_mode_set(librados::IoCtx *ioctx, cls::rbd::MirrorMode mirror_mode); +void mirror_remote_namespace_get_start(librados::ObjectReadOperation *op); +int mirror_remote_namespace_get_finish(ceph::buffer::list::const_iterator *it, + std::string *mirror_namespace); +int mirror_remote_namespace_get(librados::IoCtx *ioctx, + std::string *mirror_namespace); +int mirror_remote_namespace_set(librados::IoCtx *ioctx, + const std::string &mirror_namespace); + int mirror_peer_ping(librados::IoCtx *ioctx, const std::string& site_name, const std::string& fsid); diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h index f9af9262b2aa..3773d216bde7 100644 --- a/src/include/rbd/librbd.h +++ b/src/include/rbd/librbd.h @@ -577,6 +577,11 @@ CEPH_RBD_API int rbd_mirror_mode_get(rados_ioctx_t io_ctx, rbd_mirror_mode_t *mirror_mode); CEPH_RBD_API int rbd_mirror_mode_set(rados_ioctx_t io_ctx, rbd_mirror_mode_t mirror_mode); +CEPH_RBD_API int rbd_mirror_remote_namespace_get(rados_ioctx_t io_ctx, + char *remote_namespace, + size_t *max_len); +CEPH_RBD_API int rbd_mirror_remote_namespace_set(rados_ioctx_t io_ctx, + const char *remote_namespace); CEPH_RBD_API int rbd_mirror_uuid_get(rados_ioctx_t io_ctx, char *uuid, size_t *max_len); diff --git a/src/include/rbd/librbd.hpp b/src/include/rbd/librbd.hpp index 50a6c623d3a0..2755c4749bb3 100644 --- a/src/include/rbd/librbd.hpp +++ b/src/include/rbd/librbd.hpp @@ -357,6 +357,10 @@ class CEPH_RBD_API RBD int mirror_mode_get(IoCtx& io_ctx, rbd_mirror_mode_t *mirror_mode); int mirror_mode_set(IoCtx& io_ctx, rbd_mirror_mode_t mirror_mode); + int mirror_remote_namespace_get(IoCtx& io_ctx, + std::string* remote_namespace); + int mirror_remote_namespace_set(IoCtx& io_ctx, + const std::string& remote_namespace); int mirror_uuid_get(IoCtx& io_ctx, std::string* mirror_uuid); int mirror_peer_bootstrap_create(IoCtx& io_ctx, std::string* token); diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc index 2cfad0d32753..68ee1c2b7780 100644 --- a/src/librbd/api/Mirror.cc +++ b/src/librbd/api/Mirror.cc @@ -1108,6 +1108,7 @@ int Mirror::mode_set(librados::IoCtx& io_ctx, } } + std::string current_namespace; cls::rbd::MirrorMode current_mirror_mode; r = cls_client::mirror_mode_get(&io_ctx, ¤t_mirror_mode); if (r < 0) { @@ -1115,9 +1116,8 @@ int Mirror::mode_set(librados::IoCtx& io_ctx, << dendl; return r; } - if (current_mirror_mode == next_mirror_mode) { - return 0; + return 0; // Nothing more to be done } else if (current_mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) { uuid_d uuid_gen; uuid_gen.generate_random(); @@ -1271,6 +1271,47 @@ int Mirror::mode_set(librados::IoCtx& io_ctx, return 0; } +template +int Mirror::remote_namespace_get(librados::IoCtx& io_ctx, + std::string* remote_namespace) { + + CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << dendl; + + int r = cls_client::mirror_remote_namespace_get(&io_ctx, remote_namespace); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to retrieve mirroring mode: " << cpp_strerror(r) + << dendl; + return r; + } + return 0; +} + + +template +int Mirror::remote_namespace_set(librados::IoCtx& io_ctx, + const std::string& remote_namespace) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << dendl; + + std::string ns_name = io_ctx.get_namespace(); + + if (ns_name.empty() && !remote_namespace.empty()) { + lderr(cct) << "cannot set remote namespace for the default namespace." + << dendl; + return -EINVAL; + } + + int r = cls_client::mirror_remote_namespace_set(&io_ctx, remote_namespace); + if (r < 0 ) { + lderr(cct) << "failed to set mirroring mode: " << cpp_strerror(r) + << dendl; + return r; + } + return 0; +} + + template int Mirror::uuid_get(librados::IoCtx& io_ctx, std::string* mirror_uuid) { CephContext *cct = reinterpret_cast(io_ctx.cct()); diff --git a/src/librbd/api/Mirror.h b/src/librbd/api/Mirror.h index b3a552b13b7e..814845e39c77 100644 --- a/src/librbd/api/Mirror.h +++ b/src/librbd/api/Mirror.h @@ -31,6 +31,16 @@ struct Mirror { static int mode_get(librados::IoCtx& io_ctx, rbd_mirror_mode_t *mirror_mode); static int mode_set(librados::IoCtx& io_ctx, rbd_mirror_mode_t mirror_mode); + static int remote_namespace_get(librados::IoCtx& io_ctx, + std::string* remote_namespace); +/* + static int remote_namespace_get(librados::IoCtx& io_ctx, + std::string* remote_namespace, + Context* on_finish); +*/ + static int remote_namespace_set(librados::IoCtx& io_ctx, + const std::string& remote_namespace); + static int uuid_get(librados::IoCtx& io_ctx, std::string* mirror_uuid); static void uuid_get(librados::IoCtx& io_ctx, std::string* mirror_uuid, Context* on_finish); diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index ed8ec9e91305..5a876fc07ad2 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -1101,6 +1101,24 @@ namespace librbd { return librbd::api::Mirror<>::mode_set(io_ctx, mirror_mode); } +/* + int RBD::mirror_mode_set2(IoCtx& io_ctx, rbd_mirror_mode_t mirror_mode, + const std::string& ns_name) { + return librbd::api::Mirror<>::mode_set(io_ctx, mirror_mode, ns_name); + } +*/ + int RBD::mirror_remote_namespace_get(IoCtx& io_ctx, + std::string* remote_namespace) { + return librbd::api::Mirror<>::remote_namespace_get(io_ctx, + remote_namespace); + } + + int RBD::mirror_remote_namespace_set(IoCtx& io_ctx, + const std::string& remote_namespace) { + return librbd::api::Mirror<>::remote_namespace_set(io_ctx, + remote_namespace); + } + int RBD::mirror_uuid_get(IoCtx& io_ctx, std::string* mirror_uuid) { return librbd::api::Mirror<>::uuid_get(io_ctx, mirror_uuid); } @@ -3392,9 +3410,41 @@ extern "C" int rbd_mirror_mode_set(rados_ioctx_t p, rbd_mirror_mode_t mirror_mode) { librados::IoCtx io_ctx; librados::IoCtx::from_rados_ioctx_t(p, io_ctx); + //std::string ns_name = io_ctx.get_namespace(); return librbd::api::Mirror<>::mode_set(io_ctx, mirror_mode); } +extern "C" int rbd_mirror_remote_namespace_get(rados_ioctx_t p, + char *remote_namespace, + size_t *max_len) { + librados::IoCtx io_ctx; + librados::IoCtx::from_rados_ioctx_t(p, io_ctx); + + std::string remote_namespace_str; + int r = librbd::api::Mirror<>::remote_namespace_get(io_ctx, + &remote_namespace_str); + if (r < 0) { + return r; + } + + auto total_len = remote_namespace_str.size() + 1; + if (*max_len < total_len) { + *max_len = total_len; + return -ERANGE; + } + *max_len = total_len; + + strcpy(remote_namespace, remote_namespace_str.c_str()); + return 0; +} + +extern "C" int rbd_mirror_remote_namespace_set(rados_ioctx_t p, + const char *remote_namespace) { + librados::IoCtx io_ctx; + librados::IoCtx::from_rados_ioctx_t(p, io_ctx); + return librbd::api::Mirror<>::remote_namespace_set(io_ctx, remote_namespace); +} + extern "C" int rbd_mirror_uuid_get(rados_ioctx_t p, char *mirror_uuid, size_t *max_len) { librados::IoCtx io_ctx; diff --git a/src/pybind/rbd/c_rbd.pxd b/src/pybind/rbd/c_rbd.pxd index 3a960a2a885e..2c2dd42fd78e 100644 --- a/src/pybind/rbd/c_rbd.pxd +++ b/src/pybind/rbd/c_rbd.pxd @@ -394,6 +394,12 @@ cdef extern from "rbd/librbd.h" nogil: int rbd_mirror_mode_get(rados_ioctx_t io, rbd_mirror_mode_t *mirror_mode) int rbd_mirror_mode_set(rados_ioctx_t io, rbd_mirror_mode_t mirror_mode) + int rbd_mirror_remote_namespace_get(rados_ioctx_t io_ctx, + char *remote_namespace, + size_t *max_len) + int rbd_mirror_remote_namespace_set(rados_ioctx_t io_ctx, + const char *remote_namespace) + int rbd_mirror_uuid_get(rados_ioctx_t io_ctx, char *mirror_uuid, size_t *max_len) diff --git a/src/pybind/rbd/mock_rbd.pxi b/src/pybind/rbd/mock_rbd.pxi index 40280dc9c0d9..0367bb9ce37f 100644 --- a/src/pybind/rbd/mock_rbd.pxi +++ b/src/pybind/rbd/mock_rbd.pxi @@ -437,6 +437,14 @@ cdef nogil: int rbd_mirror_mode_set(rados_ioctx_t io, rbd_mirror_mode_t mirror_mode): pass + int rbd_mirror_remote_namespace_get(rados_ioctx_t io_ctx, + char *remote_namespace, + size_t *max_len): + pass + int rbd_mirror_remote_namespace_set(rados_ioctx_t io_ctx, + const char *remote_namespace): + pass + int rbd_mirror_uuid_get(rados_ioctx_t io_ctx, char *mirror_uuid, size_t *max_len): pass diff --git a/src/pybind/rbd/rbd.pyx b/src/pybind/rbd/rbd.pyx index 5290e48832d1..7cd7da774ea3 100644 --- a/src/pybind/rbd/rbd.pyx +++ b/src/pybind/rbd/rbd.pyx @@ -1330,6 +1330,52 @@ class RBD(object): if ret != 0: raise make_ex(ret, 'error setting mirror mode') + def mirror_remote_namespace_get(self, ioctx): + """ + Get mirror remote namespace + + :param ioctx: determines which RADOS pool is read + :type ioctx: :class:`rados.Ioctx` + :returns: str - mirror remote namespace + """ + cdef: + rados_ioctx_t _ioctx = convert_ioctx(ioctx) + char *_remote_namespace = NULL + size_t _max_size = 512 + try: + while True: + _remote_namespace = realloc_chk(_remote_namespace, + _max_size) + with nogil: + ret = rbd_mirror_remote_namespace_get(_ioctx, + _remote_namespace, + &_max_size) + if ret >= 0: + break + elif ret != -errno.ERANGE: + raise make_ex(ret, 'error retrieving remote namespace') + return decode_cstr(_remote_namespace) + finally: + free(_remote_namespace) + + def mirror_remote_namespace_set(self, ioctx, remote_namespace): + """ + Set the remote namespace + + :param ioctx: determines which RADOS pool is written + :type ioctx: :class:`rados.Ioctx` + :param remote_namespace: the remote cluster namespace to mirror to + :type str: + """ + remote_namespace = cstr(remote_namespace, 'remote_namespace') + cdef: + rados_ioctx_t _ioctx = convert_ioctx(ioctx) + char *_remote_namespace = remote_namespace + with nogil: + ret = rbd_mirror_remote_namespace_set(_ioctx, _remote_namespace) + if ret != 0: + raise make_ex(ret, 'error setting remote namespace') + def mirror_uuid_get(self, ioctx): """ Get pool mirror uuid diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t index ff25b5973a40..11e7090a3d16 100644 --- a/src/test/cli/rbd/help.t +++ b/src/test/cli/rbd/help.t @@ -1832,20 +1832,22 @@ rbd help mirror pool enable usage: rbd mirror pool enable [--pool ] [--namespace ] [--site-name ] - - + [--remote-namespace ] + + Enable RBD mirroring in a pool or namespace. - + Positional arguments - pool specification - (example: [/] - mirror mode [image or pool] - + pool specification + (example: [/] + mirror mode [image or pool] + Optional arguments - -p [ --pool ] arg pool name - --namespace arg namespace name - --site-name arg local site name - + -p [ --pool ] arg pool name + --namespace arg namespace name + --site-name arg local site name + --remote-namespace arg remote namespace name + rbd help mirror pool info usage: rbd mirror pool info [--pool ] [--namespace ] [--format ] [--pretty-format] [--all] diff --git a/src/test/cls_rbd/test_cls_rbd.cc b/src/test/cls_rbd/test_cls_rbd.cc index 9093b5355d51..7eb03cc42f54 100644 --- a/src/test/cls_rbd/test_cls_rbd.cc +++ b/src/test/cls_rbd/test_cls_rbd.cc @@ -1606,7 +1606,9 @@ TEST_F(TestClsRbd, mirror) { ASSERT_EQ(-ENOENT, mirror_peer_list(&ioctx, &peers)); std::string uuid; + std::string remote_ns; ASSERT_EQ(-ENOENT, mirror_uuid_get(&ioctx, &uuid)); + ASSERT_EQ(-ENOENT, mirror_remote_namespace_get(&ioctx, &remote_ns)); ASSERT_EQ(-EINVAL, mirror_peer_add(&ioctx, {"uuid1", MIRROR_PEER_DIRECTION_RX, "siteA", "client", "mirror uuid"})); @@ -1622,11 +1624,16 @@ TEST_F(TestClsRbd, mirror) { ASSERT_EQ(0, mirror_uuid_get(&ioctx, &uuid)); ASSERT_EQ("mirror-uuid", uuid); + ASSERT_EQ(0, mirror_remote_namespace_set(&ioctx, "remote-ns")); + ASSERT_EQ(0, mirror_remote_namespace_get(&ioctx, &remote_ns)); + ASSERT_EQ("remote-ns", remote_ns); + ASSERT_EQ(0, mirror_mode_set(&ioctx, cls::rbd::MIRROR_MODE_IMAGE)); ASSERT_EQ(0, mirror_mode_get(&ioctx, &mirror_mode)); ASSERT_EQ(cls::rbd::MIRROR_MODE_IMAGE, mirror_mode); ASSERT_EQ(-EINVAL, mirror_uuid_set(&ioctx, "new-mirror-uuid")); + ASSERT_EQ(-EINVAL, mirror_remote_namespace_set(&ioctx, "new-remote-ns")); ASSERT_EQ(0, mirror_mode_set(&ioctx, cls::rbd::MIRROR_MODE_POOL)); ASSERT_EQ(0, mirror_mode_get(&ioctx, &mirror_mode)); @@ -1726,6 +1733,7 @@ TEST_F(TestClsRbd, mirror) { ASSERT_EQ(0, mirror_mode_get(&ioctx, &mirror_mode)); ASSERT_EQ(cls::rbd::MIRROR_MODE_DISABLED, mirror_mode); ASSERT_EQ(-ENOENT, mirror_uuid_get(&ioctx, &uuid)); + ASSERT_EQ(-ENOENT, mirror_remote_namespace_get(&ioctx, &remote_ns)); } TEST_F(TestClsRbd, mirror_image) { diff --git a/src/test/pybind/test_rbd.py b/src/test/pybind/test_rbd.py index 9e307cbb9bf1..c2fe6648c6b9 100644 --- a/src/test/pybind/test_rbd.py +++ b/src/test/pybind/test_rbd.py @@ -2391,6 +2391,22 @@ def test_site_name(self): self.rbd.mirror_site_name_set(rados, "") eq(rados.get_fsid(), self.rbd.mirror_site_name_get(rados)) + def test_mirror_remote_namespace(self): + remote_namespace = "remote-ns" + assert_raises(InvalidArgument, self.rbd.mirror_remote_namespace_set, + ioctx, remote_namespace) + eq("", self.rbd.mirror_remote_namespace_get(ioctx)) + self.rbd.namespace_create(ioctx, "ns1") + ioctx.set_namespace("ns1") + self.rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_IMAGE) + assert_raises(InvalidArgument, self.rbd.mirror_remote_namespace_set, + ioctx, remote_namespace) + self.rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_DISABLED) + self.rbd.mirror_remote_namespace_set(ioctx, remote_namespace) + eq(remote_namespace, self.rbd.mirror_remote_namespace_get(ioctx)) + ioctx.set_namespace("") + self.rbd.namespace_remove(ioctx, "ns1") + def test_mirror_peer_bootstrap(self): eq([], list(self.rbd.mirror_peer_list(ioctx))) diff --git a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc index ece1a3396115..52aefa90a8b7 100644 --- a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc +++ b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc @@ -409,7 +409,7 @@ TEST_F(TestMockNamespaceReplayer, Init_LocalMirrorStatusUpdaterError) { expect_mirror_status_updater_init(*mock_local_mirror_status_updater, -EINVAL); MockNamespaceReplayer namespace_replayer( - {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", + {}, {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, nullptr, nullptr, nullptr, nullptr, nullptr); @@ -432,7 +432,7 @@ TEST_F(TestMockNamespaceReplayer, Init_RemoteMirrorStatusUpdaterError) { expect_mirror_status_updater_shut_down(*mock_local_mirror_status_updater); MockNamespaceReplayer namespace_replayer( - {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", + {}, {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, nullptr, nullptr, nullptr, nullptr, nullptr); @@ -458,7 +458,7 @@ TEST_F(TestMockNamespaceReplayer, Init_InstanceReplayerError) { expect_mirror_status_updater_shut_down(*mock_local_mirror_status_updater); MockNamespaceReplayer namespace_replayer( - {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", + {}, {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, nullptr, nullptr, nullptr, nullptr, nullptr); @@ -489,7 +489,7 @@ TEST_F(TestMockNamespaceReplayer, Init_InstanceWatcherError) { expect_mirror_status_updater_shut_down(*mock_local_mirror_status_updater); MockNamespaceReplayer namespace_replayer( - {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", + {}, {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, nullptr, nullptr, nullptr, nullptr, nullptr); @@ -517,7 +517,7 @@ TEST_F(TestMockNamespaceReplayer, Init) { MockServiceDaemon mock_service_daemon; MockNamespaceReplayer namespace_replayer( - {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", + {}, {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, nullptr, nullptr, &mock_service_daemon, nullptr, nullptr); @@ -557,7 +557,7 @@ TEST_F(TestMockNamespaceReplayer, AcquireLeader) { MockServiceDaemon mock_service_daemon; MockNamespaceReplayer namespace_replayer( - {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", + {}, {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, nullptr, nullptr, &mock_service_daemon, nullptr, nullptr); diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc index 69fe79931481..4b65b0c33c21 100644 --- a/src/test/rbd_mirror/test_mock_PoolReplayer.cc +++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc @@ -129,7 +129,8 @@ struct NamespaceReplayer { static std::map s_instances; static NamespaceReplayer *create( - const std::string &name, + const std::string &local_name, + const std::string &remote_name, librados::IoCtx &local_ioctx, librados::IoCtx &remote_ioctx, const std::string &local_mirror_uuid, @@ -141,9 +142,9 @@ struct NamespaceReplayer { ServiceDaemon *service_daemon, journal::CacheManagerHandler *cache_manager_handler, PoolMetaCache* pool_meta_cache) { - ceph_assert(s_instances.count(name)); - auto namespace_replayer = s_instances[name]; - s_instances.erase(name); + ceph_assert(s_instances.count(local_name)); + auto namespace_replayer = s_instances[local_name]; + s_instances.erase(local_name); return namespace_replayer; } @@ -363,6 +364,17 @@ class TestMockPoolReplayer : public TestMockFixture { Return(0))); } + void expect_mirror_namespace_get(librados::MockTestMemIoCtxImpl *io_ctx_impl, + const std::string &remote_namespace, int r) { + EXPECT_CALL(*io_ctx_impl, + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_namespace_get"), + _, _, _, _)) + .WillRepeatedly(DoAll(WithArg<5>(Invoke([remote_namespace](bufferlist *bl) { + encode(remote_namespace, *bl); + })), + Return(r))); + } + void expect_clone(librados::MockTestMemIoCtxImpl* mock_io_ctx) { EXPECT_CALL(*mock_io_ctx, clone()) .WillRepeatedly(Invoke([mock_io_ctx]() { @@ -737,6 +749,7 @@ TEST_F(TestMockPoolReplayer, Namespaces) { expect_clone(mock_local_io_ctx); expect_mirror_mode_get(mock_local_io_ctx); + expect_mirror_namespace_get(mock_local_io_ctx, "", -ENOENT); InSequence seq; @@ -856,6 +869,7 @@ TEST_F(TestMockPoolReplayer, NamespacesError) { expect_clone(mock_local_io_ctx); expect_mirror_mode_get(mock_local_io_ctx); + expect_mirror_namespace_get(mock_local_io_ctx, "", -ENOENT); InSequence seq; diff --git a/src/tools/rbd/ArgumentTypes.h b/src/tools/rbd/ArgumentTypes.h index cc7c48136369..3c28acf8cee2 100644 --- a/src/tools/rbd/ArgumentTypes.h +++ b/src/tools/rbd/ArgumentTypes.h @@ -55,6 +55,7 @@ static const std::string DEST_SNAPSHOT_NAME("dest-snap"); static const std::string PATH("path"); static const std::string FROM_SNAPSHOT_NAME("from-snap"); static const std::string WHOLE_OBJECT("whole-object"); +static const std::string REMOTE_NAMESPACE_NAME("remote-namespace"); // encryption arguments static const std::string ENCRYPTION_FORMAT("encryption-format"); diff --git a/src/tools/rbd/action/MirrorPool.cc b/src/tools/rbd/action/MirrorPool.cc index a7877870a04a..a7187b8b5a04 100644 --- a/src/tools/rbd/action/MirrorPool.cc +++ b/src/tools/rbd/action/MirrorPool.cc @@ -1242,11 +1242,17 @@ void get_enable_arguments(po::options_description *positional, positional->add_options() ("mode", "mirror mode [image or pool]"); add_site_name_optional(options); + + options->add_options() + (at::REMOTE_NAMESPACE_NAME.c_str(), po::value(), + "remote namespace name"); } int execute_enable_disable(librados::IoCtx& io_ctx, rbd_mirror_mode_t next_mirror_mode, - const std::string &mode, bool ignore_no_update) { + const std::string &mode, + const std::string &remote_namespace, + bool ignore_no_update) { librbd::RBD rbd; rbd_mirror_mode_t current_mirror_mode; int r = rbd.mirror_mode_get(io_ctx, ¤t_mirror_mode); @@ -1276,6 +1282,13 @@ int execute_enable_disable(librados::IoCtx& io_ctx, << std::endl; } + if (current_mirror_mode == RBD_MIRROR_MODE_DISABLED) { + r = rbd.mirror_remote_namespace_set(io_ctx, remote_namespace); + if (r < 0) { + return r; + } + } + r = rbd.mirror_mode_set(io_ctx, next_mirror_mode); if (r < 0) { return r; @@ -1303,13 +1316,14 @@ int execute_disable(const po::variables_map &vm, } return execute_enable_disable(io_ctx, RBD_MIRROR_MODE_DISABLED, "disabled", - false); + "", false); } int execute_enable(const po::variables_map &vm, const std::vector &ceph_global_init_args) { std::string pool_name; std::string namespace_name; + std::string remote_namespace; size_t arg_index = 0; int r = utils::get_pool_and_namespace_names(vm, true, &pool_name, &namespace_name, &arg_index); @@ -1337,6 +1351,17 @@ int execute_enable(const po::variables_map &vm, } bool updated = false; + if (vm.count(at::REMOTE_NAMESPACE_NAME)) { + remote_namespace = vm[at::REMOTE_NAMESPACE_NAME].as(); + std::cout << " DBG: Found remote namespace : " << remote_namespace << std::endl; //TODO: remove this + if (namespace_name.empty() && !remote_namespace.empty()) { + std::cerr << "rbd: cannot mirror the default namespace to non-default." << std::endl; + return -EINVAL; + } + } else { + remote_namespace = namespace_name; + } + if (vm.count(SITE_NAME)) { librbd::RBD rbd; @@ -1351,7 +1376,7 @@ int execute_enable(const po::variables_map &vm, } } - return execute_enable_disable(io_ctx, mirror_mode, mode, updated); + return execute_enable_disable(io_ctx, mirror_mode, mode, remote_namespace, updated); } void get_info_arguments(po::options_description *positional, @@ -1366,6 +1391,8 @@ int execute_info(const po::variables_map &vm, const std::vector &ceph_global_init_args) { std::string pool_name; std::string namespace_name; + std::string remote_namespace; + std::string mirror_uuid; size_t arg_index = 0; int r = utils::get_pool_and_namespace_names(vm, false, &pool_name, &namespace_name, &arg_index); @@ -1405,8 +1432,22 @@ int execute_info(const po::variables_map &vm, if (r < 0) { return r; } + } else { + r = rbd.mirror_remote_namespace_get(io_ctx, &remote_namespace); + if (r < 0 && r != -EOPNOTSUPP && r != -ENOENT ) { + return r; + } + if (r < 0) { + remote_namespace = namespace_name; + } } + if (mirror_mode != RBD_MIRROR_MODE_DISABLED) { + r = rbd.mirror_uuid_get(io_ctx, &mirror_uuid); + if (r < 0) { + return r; + } + } std::string mirror_mode_desc; switch (mirror_mode) { case RBD_MIRROR_MODE_DISABLED: @@ -1430,18 +1471,28 @@ int execute_info(const po::variables_map &vm, std::cout << "Mode: " << mirror_mode_desc << std::endl; } - if (mirror_mode != RBD_MIRROR_MODE_DISABLED && namespace_name.empty()) { + if (mirror_mode != RBD_MIRROR_MODE_DISABLED) { + if (namespace_name.empty()) { + if (formatter != nullptr) { + formatter->dump_string("site_name", site_name); + } else { + std::cout << "Site Name: " << site_name << std::endl; + } + } if (formatter != nullptr) { - formatter->dump_string("site_name", site_name); + formatter->dump_string("mirror_uuid", mirror_uuid); + formatter->dump_string("remote_namespace", remote_namespace); } else { - std::cout << "Site Name: " << site_name << std::endl - << std::endl; + std::cout << "Mirror UUID: " << mirror_uuid << std::endl; + std::cout << "Remote Namespace: " << remote_namespace << std::endl + << std::endl; } - - r = format_mirror_peers(io_ctx, formatter, mirror_peers, + if (namespace_name.empty()) { + r = format_mirror_peers(io_ctx, formatter, mirror_peers, vm[ALL_NAME].as()); - if (r < 0) { - return r; + if (r < 0) { + return r; + } } } if (formatter != nullptr) { diff --git a/src/tools/rbd_mirror/NamespaceReplayer.cc b/src/tools/rbd_mirror/NamespaceReplayer.cc index d305d8472151..38274a542414 100644 --- a/src/tools/rbd_mirror/NamespaceReplayer.cc +++ b/src/tools/rbd_mirror/NamespaceReplayer.cc @@ -36,7 +36,8 @@ const std::string SERVICE_DAEMON_REMOTE_COUNT_KEY("image_remote_count"); template NamespaceReplayer::NamespaceReplayer( - const std::string &name, + const std::string &local_name, + const std::string &remote_name, librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx, const std::string &local_mirror_uuid, const std::string& local_mirror_peer_uuid, @@ -47,7 +48,8 @@ NamespaceReplayer::NamespaceReplayer( ServiceDaemon *service_daemon, journal::CacheManagerHandler *cache_manager_handler, PoolMetaCache* pool_meta_cache) : - m_namespace_name(name), + m_local_namespace_name(local_name), + m_remote_namespace_name(remote_name), m_local_mirror_uuid(local_mirror_uuid), m_local_mirror_peer_uuid(local_mirror_peer_uuid), m_remote_pool_meta(remote_pool_meta), @@ -57,16 +59,19 @@ NamespaceReplayer::NamespaceReplayer( m_cache_manager_handler(cache_manager_handler), m_pool_meta_cache(pool_meta_cache), m_lock(ceph::make_mutex(librbd::util::unique_lock_name( - "rbd::mirror::NamespaceReplayer " + name, this))), + "rbd::mirror::NamespaceReplayer " + local_name, this))), m_local_pool_watcher_listener(this, true), m_remote_pool_watcher_listener(this, false), m_image_map_listener(this) { - dout(10) << name << dendl; + dout(10) << "local_name=" << local_name + << ", remote_name=" << remote_name + << ", local_mirror_uuid=" << m_local_mirror_uuid + << dendl; m_local_io_ctx.dup(local_io_ctx); - m_local_io_ctx.set_namespace(name); + m_local_io_ctx.set_namespace(local_name); m_remote_io_ctx.dup(remote_io_ctx); - m_remote_io_ctx.set_namespace(name); + m_remote_io_ctx.set_namespace(remote_name); } template @@ -88,7 +93,8 @@ void NamespaceReplayer::init(Context *on_finish) { ceph_assert(m_on_finish == nullptr); m_on_finish = on_finish; - init_local_status_updater(); + get_remote_mirror_namespace(); +// init_local_status_updater(); } @@ -856,6 +862,57 @@ void NamespaceReplayer::handle_remove_image(const std::string &mirror_uuid, mirror_uuid, on_finish); } +template +void NamespaceReplayer::get_remote_mirror_namespace() { + dout(15) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_remote_namespace_get_start(&op); + + m_out_bl.clear(); + auto aio_comp = librbd::util::create_rados_callback< + NamespaceReplayer, &NamespaceReplayer::handle_get_remote_mirror_namespace>(this); + int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + + +template +void NamespaceReplayer::handle_get_remote_mirror_namespace(int r) { + dout(20) << "r=" << r << dendl; + std::string remote_namespace; + + std::lock_guard locker{m_lock}; + + if (r >= 0) { + auto it = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_remote_namespace_get_finish(&it, + &remote_namespace); + } + if (r < 0 ) { + if (r == -EOPNOTSUPP || r == -ENOENT) { + remote_namespace = m_remote_namespace_name; + } else { + derr << "failed to retrieve remote namespace: " << cpp_strerror(r) + << dendl; + ceph_assert(m_on_finish != nullptr); + m_threads->work_queue->queue(m_on_finish, r); + m_on_finish = nullptr; + return; + } + } + if (remote_namespace != m_local_namespace_name) { + derr << "remote namespace does not match: " << cpp_strerror(r) + << dendl; + ceph_assert(m_on_finish != nullptr); + m_threads->work_queue->queue(m_on_finish, -EINVAL); + m_on_finish = nullptr; + return; + } + init_local_status_updater(); +} + } // namespace mirror } // namespace rbd diff --git a/src/tools/rbd_mirror/NamespaceReplayer.h b/src/tools/rbd_mirror/NamespaceReplayer.h index e304b8253f85..02a101fd79d3 100644 --- a/src/tools/rbd_mirror/NamespaceReplayer.h +++ b/src/tools/rbd_mirror/NamespaceReplayer.h @@ -43,7 +43,8 @@ template class NamespaceReplayer { public: static NamespaceReplayer *create( - const std::string &name, + const std::string &local_name, + const std::string &remote_name, librados::IoCtx &local_ioctx, librados::IoCtx &remote_ioctx, const std::string &local_mirror_uuid, @@ -55,7 +56,7 @@ class NamespaceReplayer { ServiceDaemon *service_daemon, journal::CacheManagerHandler *cache_manager_handler, PoolMetaCache* pool_meta_cache) { - return new NamespaceReplayer(name, local_ioctx, remote_ioctx, + return new NamespaceReplayer(local_name, remote_name, local_ioctx, remote_ioctx, local_mirror_uuid, local_mirror_peer_uuid, remote_pool_meta, threads, image_sync_throttler, image_deletion_throttler, @@ -63,7 +64,8 @@ class NamespaceReplayer { pool_meta_cache); } - NamespaceReplayer(const std::string &name, + NamespaceReplayer(const std::string &local_name, + const std::string &remote_name, librados::IoCtx &local_ioctx, librados::IoCtx &remote_ioctx, const std::string &local_mirror_uuid, @@ -204,6 +206,10 @@ class NamespaceReplayer { const std::string &description, RadosRef *rados_ref, bool strip_cluster_overrides); + + void get_remote_mirror_namespace(); + void handle_get_remote_mirror_namespace(int r); + void init_local_status_updater(); void handle_init_local_status_updater(int r); @@ -264,7 +270,8 @@ class NamespaceReplayer { const std::string &instance_id, Context* on_finish); - std::string m_namespace_name; + std::string m_local_namespace_name; + std::string m_remote_namespace_name; librados::IoCtx m_local_io_ctx; librados::IoCtx m_remote_io_ctx; std::string m_local_mirror_uuid; @@ -278,6 +285,7 @@ class NamespaceReplayer { PoolMetaCache* m_pool_meta_cache; mutable ceph::mutex m_lock; + bufferlist m_out_bl; int m_ret_val = 0; Context *m_on_finish = nullptr; diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc index dcbedc67e423..bf6ed91225c9 100644 --- a/src/tools/rbd_mirror/PoolReplayer.cc +++ b/src/tools/rbd_mirror/PoolReplayer.cc @@ -369,7 +369,7 @@ void PoolReplayer::init(const std::string& site_name) { m_local_io_ctx.get_id(), {m_local_mirror_uuid}); m_default_namespace_replayer.reset(NamespaceReplayer::create( - "", m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid, + "", "", m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid, m_remote_pool_meta, m_threads, m_image_sync_throttler.get(), m_image_deletion_throttler.get(), m_service_daemon, m_cache_manager_handler, m_pool_meta_cache)); @@ -638,7 +638,7 @@ void PoolReplayer::update_namespace_replayers() { ceph_assert(ceph_mutex_is_locked(m_lock)); - std::set mirroring_namespaces; + std::map mirroring_namespaces; if (!m_stopping) { int r = list_mirroring_namespaces(&mirroring_namespaces); if (r < 0) { @@ -652,7 +652,8 @@ void PoolReplayer::update_namespace_replayers() { for (auto it = m_namespace_replayers.begin(); it != m_namespace_replayers.end(); ) { auto iter = mirroring_namespaces.find(it->first); - if (iter == mirroring_namespaces.end()) { + if (iter == mirroring_namespaces.end() || + m_namespace_map[it->first] != iter->second) { auto namespace_replayer = it->second; auto on_shut_down = new LambdaContext( [namespace_replayer, ctx=gather_ctx->new_sub()](int r) { @@ -661,6 +662,7 @@ void PoolReplayer::update_namespace_replayers() { }); m_service_daemon->remove_namespace(m_local_pool_id, it->first); namespace_replayer->shut_down(on_shut_down); + m_namespace_map.erase(it->first); it = m_namespace_replayers.erase(it); } else { mirroring_namespaces.erase(iter); @@ -670,7 +672,7 @@ void PoolReplayer::update_namespace_replayers() { for (auto &name : mirroring_namespaces) { auto namespace_replayer = NamespaceReplayer::create( - name, m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid, + name.first, name.second, m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid, m_remote_pool_meta, m_threads, m_image_sync_throttler.get(), m_image_deletion_throttler.get(), m_service_daemon, m_cache_manager_handler, m_pool_meta_cache); @@ -682,10 +684,11 @@ void PoolReplayer::update_namespace_replayers() { derr << "failed to initialize namespace replayer for namespace " << name << ": " << cpp_strerror(r) << dendl; delete namespace_replayer; - mirroring_namespaces.erase(name); + mirroring_namespaces.erase(name.first); } else { - m_namespace_replayers[name] = namespace_replayer; - m_service_daemon->add_namespace(m_local_pool_id, name); + m_namespace_replayers[name.first] = namespace_replayer; + m_namespace_map[name.first] = name.second; + m_service_daemon->add_namespace(m_local_pool_id, name.first); } ctx->complete(r); }); @@ -703,7 +706,7 @@ void PoolReplayer::update_namespace_replayers() { auto acquire_gather_ctx = new C_Gather(cct, &acquire_cond); for (auto &name : mirroring_namespaces) { - namespace_replayer_acquire_leader(name, acquire_gather_ctx->new_sub()); + namespace_replayer_acquire_leader(name.first, acquire_gather_ctx->new_sub()); } acquire_gather_ctx->activate(); @@ -715,7 +718,7 @@ void PoolReplayer::update_namespace_replayers() { m_leader_watcher->list_instances(&instance_ids); for (auto &name : mirroring_namespaces) { - auto it = m_namespace_replayers.find(name); + auto it = m_namespace_replayers.find(name.first); if (it == m_namespace_replayers.end()) { // acquire leader for this namespace replayer failed continue; @@ -726,7 +729,7 @@ void PoolReplayer::update_namespace_replayers() { std::string leader_instance_id; if (m_leader_watcher->get_leader_instance_id(&leader_instance_id)) { for (auto &name : mirroring_namespaces) { - m_namespace_replayers[name]->handle_update_leader(leader_instance_id); + m_namespace_replayers[name.first]->handle_update_leader(leader_instance_id); } } } @@ -734,7 +737,7 @@ void PoolReplayer::update_namespace_replayers() { template int PoolReplayer::list_mirroring_namespaces( - std::set *namespaces) { + std::map *namespaces) { dout(20) << dendl; ceph_assert(ceph_mutex_is_locked(m_lock)); @@ -751,6 +754,7 @@ int PoolReplayer::list_mirroring_namespaces( ns_ioctx.dup(m_local_io_ctx); ns_ioctx.set_namespace(name); + std::string mirror_ns; cls::rbd::MirrorMode mirror_mode = cls::rbd::MIRROR_MODE_DISABLED; int r = librbd::cls_client::mirror_mode_get(&ns_ioctx, &mirror_mode); if (r < 0 && r != -ENOENT) { @@ -763,8 +767,22 @@ int PoolReplayer::list_mirroring_namespaces( dout(10) << "mirroring is disabled for namespace " << name << dendl; continue; } - - namespaces->insert(name); + r = librbd::cls_client::mirror_remote_namespace_get(&ns_ioctx, &mirror_ns); + if (r < 0) { + + dout(10) << " getting remote namespace failed: " << name << dendl; + if (r != -ENOENT) { + derr << "failed to get remote mirror namespace: " << cpp_strerror(r) + << dendl; + continue; + } else { + mirror_ns = name; + } + } + + dout(10) << " local namespace: " << name << ", remote namespace:" + << mirror_ns << dendl; + namespaces->insert(std::make_pair(name, mirror_ns)); } return 0; @@ -806,6 +824,7 @@ void PoolReplayer::namespace_replayer_acquire_leader(const std::string &name, auto namespace_replayer = m_namespace_replayers[name]; m_namespace_replayers.erase(name); + m_namespace_map.erase(name); auto on_shut_down = new LambdaContext( [namespace_replayer, on_finish](int r) { delete namespace_replayer; diff --git a/src/tools/rbd_mirror/PoolReplayer.h b/src/tools/rbd_mirror/PoolReplayer.h index e0fd753778d9..38d2f08ba846 100644 --- a/src/tools/rbd_mirror/PoolReplayer.h +++ b/src/tools/rbd_mirror/PoolReplayer.h @@ -107,7 +107,7 @@ class PoolReplayer { bool strip_cluster_overrides); void update_namespace_replayers(); - int list_mirroring_namespaces(std::set *namespaces); + int list_mirroring_namespaces(std::map *namespaces); void namespace_replayer_acquire_leader(const std::string &name, Context *on_finish); @@ -221,6 +221,7 @@ class PoolReplayer { std::unique_ptr> m_default_namespace_replayer; std::map *> m_namespace_replayers; + std::map m_namespace_map; std::string m_asok_hook_name; AdminSocketHook *m_asok_hook = nullptr;