rbd-mirror: allow mirroring to a different namespace

Allows a namespace in a pool to be mirrored to a differently named namespace in the secondary cluster. Signed-off-by: N Balachandran <[email protected]>
ceph · Sep 5, 2024 · f9d030a · f9d030a
1 parent a71318b
commit f9d030a
Show file tree

Hide file tree

Showing 28 changed files with 641 additions and 138 deletions.
diff --git a/doc/man/8/rbd.rst b/doc/man/8/rbd.rst
@@ -543,8 +543,9 @@ Commands
 
 :command:`mirror pool info` [*pool-name*]
   Show information about the pool or namespace mirroring configuration.
-  For a pool, it includes mirroring mode, peer UUID, remote cluster name,
-  and remote client name. For a namespace, it includes only mirroring mode.
+  For both pools and namespaces, it includes the mirroring mode, mirror uuid
+  and remote namespace. For pools, it additonally includes the site name,
+  peer uuid, remote cluster name, and remote client name.
 
 :command:`mirror pool peer add` [*pool-name*] *remote-cluster-spec*
   Add a mirroring peer to a pool.

diff --git a/qa/workunits/rbd/rbd_mirror.sh b/qa/workunits/rbd/rbd_mirror.sh
diff --git a/qa/workunits/rbd/rbd_mirror_bootstrap.sh b/qa/workunits/rbd/rbd_mirror_bootstrap.sh
@@ -38,7 +38,7 @@ create_image_and_enable_mirror ${CLUSTER1} ${POOL} image1
 
 wait_for_image_replay_started ${CLUSTER2} ${POOL} image1
 write_image ${CLUSTER1} ${POOL} image1 100
-wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} image1
+wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${POOL} image1
 wait_for_replaying_status_in_pool_dir ${CLUSTER2} ${POOL} image1
 
 testlog "TEST: verify rx-tx direction"
@@ -54,12 +54,12 @@ enable_mirror ${CLUSTER2} ${PARENT_POOL} image2
 
 wait_for_image_replay_started ${CLUSTER2} ${PARENT_POOL} image1
 write_image ${CLUSTER1} ${PARENT_POOL} image1 100
-wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${PARENT_POOL} image1
+wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${PARENT_POOL} ${PARENT_POOL} image1
 wait_for_replaying_status_in_pool_dir ${CLUSTER2} ${PARENT_POOL} image1
 
 wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} image2
 write_image ${CLUSTER2} ${PARENT_POOL} image2 100
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} image2
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${PARENT_POOL} image2
 wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} image2
 
 testlog "TEST: pool replayer and callout cleanup when peer is updated"

diff --git a/qa/workunits/rbd/rbd_mirror_ha.sh b/qa/workunits/rbd/rbd_mirror_ha.sh
@@ -79,7 +79,7 @@ test_replay()
 	    wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} \
 					'down+unknown'
 	fi
-	compare_images ${POOL} ${image}
+	compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image}
     done
 }
 

diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh
@@ -743,17 +743,18 @@ wait_for_journal_replay_complete()
 {
     local local_cluster=$1
     local cluster=$2
-    local pool=$3
-    local image=$4
+    local local_pool=$3
+    local remote_pool=$4
+    local image=$5
     local s master_pos mirror_pos last_mirror_pos
     local master_tag master_entry mirror_tag mirror_entry
 
     while true; do
         for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do
             sleep ${s}
-            flush "${local_cluster}" "${pool}" "${image}"
-            master_pos=$(get_master_journal_position "${cluster}" "${pool}" "${image}")
-            mirror_pos=$(get_mirror_journal_position "${cluster}" "${pool}" "${image}")
+            flush "${local_cluster}" "${local_pool}" "${image}"
+            master_pos=$(get_master_journal_position "${cluster}" "${remote_pool}" "${image}")
+            mirror_pos=$(get_mirror_journal_position "${cluster}" "${remote_pool}" "${image}")
             test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0
             test "${mirror_pos}" != "${last_mirror_pos}" && break
         done
@@ -796,21 +797,22 @@ wait_for_snapshot_sync_complete()
 {
     local local_cluster=$1
     local cluster=$2
-    local pool=$3
-    local image=$4
+    local local_pool=$3
+    local remote_pool=$4
+    local image=$5
 
-    local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}-${image}.status)
-    local local_status_log=${TEMPDIR}/$(mkfname ${local_cluster}-${pool}-${image}.status)
+    local status_log=${TEMPDIR}/$(mkfname ${cluster}-${remote_pool}-${image}.status)
+    local local_status_log=${TEMPDIR}/$(mkfname ${local_cluster}-${local_pool}-${image}.status)
 
-    mirror_image_snapshot "${cluster}" "${pool}" "${image}"
-    get_newest_mirror_snapshot "${cluster}" "${pool}" "${image}" "${status_log}"
+    mirror_image_snapshot "${cluster}" "${remote_pool}" "${image}"
+    get_newest_mirror_snapshot "${cluster}" "${remote_pool}" "${image}" "${status_log}"
     local snapshot_id=$(xmlstarlet sel -t -v "//snapshot/id" < ${status_log})
 
     while true; do
         for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do
             sleep ${s}
 
-            get_newest_mirror_snapshot "${local_cluster}" "${pool}" "${image}" "${local_status_log}"
+            get_newest_mirror_snapshot "${local_cluster}" "${local_pool}" "${image}" "${local_status_log}"
             local primary_snapshot_id=$(xmlstarlet sel -t -v "//snapshot/namespace/primary_snap_id" < ${local_status_log})
 
             test "${snapshot_id}" = "${primary_snapshot_id}" && return 0
@@ -825,13 +827,14 @@ wait_for_replay_complete()
 {
     local local_cluster=$1
     local cluster=$2
-    local pool=$3
-    local image=$4
+    local local_pool=$3
+    local remote_pool=$4
+    local image=$5
 
     if [ "${RBD_MIRROR_MODE}" = "journal" ]; then
-        wait_for_journal_replay_complete ${local_cluster} ${cluster} ${pool} ${image}
+        wait_for_journal_replay_complete ${local_cluster} ${cluster} ${local_pool} ${remote_pool} ${image}
     elif [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
-        wait_for_snapshot_sync_complete ${local_cluster} ${cluster} ${pool} ${image}
+        wait_for_snapshot_sync_complete ${local_cluster} ${cluster} ${local_pool} ${remote_pool} ${image}
     else
         return 1
     fi
@@ -1298,16 +1301,19 @@ show_diff()
 
 compare_images()
 {
-    local pool=$1
-    local image=$2
     local ret=0
+    local local_cluster=$1
+    local cluster=$2
+    local local_pool=$3
+    local remote_pool=$4
+    local image=$5
 
-    local rmt_export=${TEMPDIR}/$(mkfname ${CLUSTER2}-${pool}-${image}.export)
-    local loc_export=${TEMPDIR}/$(mkfname ${CLUSTER1}-${pool}-${image}.export)
+    local rmt_export=${TEMPDIR}/$(mkfname ${cluster}-${remote_pool}-${image}.export)
+    local loc_export=${TEMPDIR}/$(mkfname ${local_cluster}-${local_pool}-${image}.export)
 
     rm -f ${rmt_export} ${loc_export}
-    rbd --cluster ${CLUSTER2} export ${pool}/${image} ${rmt_export}
-    rbd --cluster ${CLUSTER1} export ${pool}/${image} ${loc_export}
+    rbd --cluster ${cluster} export ${remote_pool}/${image} ${rmt_export}
+    rbd --cluster ${local_cluster} export ${local_pool}/${image} ${loc_export}
     if ! cmp ${rmt_export} ${loc_export}
     then
         show_diff ${rmt_export} ${loc_export}

diff --git a/qa/workunits/rbd/rbd_mirror_stress.sh b/qa/workunits/rbd/rbd_mirror_stress.sh
@@ -111,7 +111,7 @@ do
   snap_name="snap${i}"
   create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name}
   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image}
   wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name}
 
   if [ -n "${clean_snap_name}" ]; then
@@ -124,7 +124,7 @@ do
 done
 
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image}
 wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${clean_snap_name}
 
 for i in `seq 1 10`
@@ -173,7 +173,7 @@ do
   image="image_${i}"
   create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name}
   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image}
   wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name}
   compare_image_snaps ${POOL} ${image} ${snap_name}
 done

diff --git a/src/cls/rbd/cls_rbd.cc b/src/cls/rbd/cls_rbd.cc
@@ -4624,6 +4624,7 @@ static const std::string STATUS_GLOBAL_KEY_PREFIX("status_global_");
 static const std::string REMOTE_STATUS_GLOBAL_KEY_PREFIX("remote_status_global_");
 static const std::string INSTANCE_KEY_PREFIX("instance_");
 static const std::string MIRROR_IMAGE_MAP_KEY_PREFIX("image_map_");
+static const std::string REMOTE_NAMESPACE("remote_namespace");
 
 std::string peer_key(const std::string &uuid) {
   return PEER_KEY_PREFIX + uuid;
@@ -5920,6 +5921,54 @@ int mirror_mode_set(cls_method_context_t hctx, bufferlist *in,
     if (r < 0) {
       return r;
     }
+
+    r = remove_key(hctx, mirror::REMOTE_NAMESPACE);
+    if (r < 0) {
+      return r;
+    }
+  }
+  return 0;
+}
+
+int mirror_namespace_get(cls_method_context_t hctx, bufferlist *in,
+                         bufferlist *out) {
+  std::string mirror_ns_decode;
+  int r = read_key(hctx, mirror::REMOTE_NAMESPACE, &mirror_ns_decode);
+  if (r < 0) {
+    CLS_ERR("error getting mirror namespace: %s", cpp_strerror(r).c_str());
+    return r;
+  }
+
+  encode(mirror_ns_decode, *out);
+  return 0;
+}
+
+int mirror_namespace_set(cls_method_context_t hctx, bufferlist *in,
+                         bufferlist *out) {
+  std::string mirror_namespace;
+  try {
+    auto bl_it = in->cbegin();
+    decode(mirror_namespace, bl_it);
+  } catch (const ceph::buffer::error &err) {
+    return -EINVAL;
+  }
+
+  uint32_t mirror_mode;
+  int r = read_key(hctx, mirror::MODE, &mirror_mode);
+  if (r < 0 && r != -ENOENT) {
+    return r;
+  } else if (r == 0 && mirror_mode != cls::rbd::MIRROR_MODE_DISABLED) {
+    CLS_ERR("cannot set mirror remote namespace while mirroring enabled");
+    return -EINVAL;
+  }
+
+  bufferlist bl;
+  encode(mirror_namespace, bl);
+
+  r = cls_cxx_map_set_val(hctx, mirror::REMOTE_NAMESPACE, &bl);
+  if (r < 0) {
+    CLS_ERR("error setting mirror namespace: %s", cpp_strerror(r).c_str());
+    return r;
   }
   return 0;
 }
@@ -8278,6 +8327,8 @@ CLS_INIT(rbd)
   cls_method_handle_t h_mirror_uuid_set;
   cls_method_handle_t h_mirror_mode_get;
   cls_method_handle_t h_mirror_mode_set;
+  cls_method_handle_t h_mirror_namespace_get;
+  cls_method_handle_t h_mirror_namespace_set;
   cls_method_handle_t h_mirror_peer_ping;
   cls_method_handle_t h_mirror_peer_list;
   cls_method_handle_t h_mirror_peer_add;
@@ -8575,6 +8626,11 @@ CLS_INIT(rbd)
   cls_register_cxx_method(h_class, "mirror_mode_set",
                           CLS_METHOD_RD | CLS_METHOD_WR,
                           mirror_mode_set, &h_mirror_mode_set);
+  cls_register_cxx_method(h_class, "mirror_namespace_get", CLS_METHOD_RD,
+                          mirror_namespace_get, &h_mirror_namespace_get);
+  cls_register_cxx_method(h_class, "mirror_namespace_set",
+                          CLS_METHOD_RD | CLS_METHOD_WR,
+                          mirror_namespace_set, &h_mirror_namespace_set);
   cls_register_cxx_method(h_class, "mirror_peer_ping",
                           CLS_METHOD_RD | CLS_METHOD_WR,
                           mirror_peer_ping, &h_mirror_peer_ping);

diff --git a/src/cls/rbd/cls_rbd_client.cc b/src/cls/rbd/cls_rbd_client.cc
@@ -1882,6 +1882,54 @@ int mirror_mode_set(librados::IoCtx *ioctx,
   return 0;
 }
 
+void mirror_remote_namespace_get_start(librados::ObjectReadOperation *op) {
+  bufferlist bl;
+  op->exec("rbd", "mirror_namespace_get", bl);
+}
+
+int mirror_remote_namespace_get_finish(bufferlist::const_iterator *it,
+                                       std::string *mirror_namespace) {
+  try {
+    decode(*mirror_namespace, *it);
+  } catch (const ceph::buffer::error &err) {
+    return -EBADMSG;
+  }
+  return 0;
+}
+
+int mirror_remote_namespace_get(librados::IoCtx *ioctx,
+			        std::string *mirror_namespace) {
+  librados::ObjectReadOperation op;
+  mirror_remote_namespace_get_start(&op);
+
+  bufferlist out_bl;
+  int r = ioctx->operate(RBD_MIRRORING, &op, &out_bl);
+  if (r < 0) {
+    return r;
+  }
+
+  auto it = out_bl.cbegin();
+  r = mirror_remote_namespace_get_finish(&it, mirror_namespace);
+  if (r < 0) {
+    return r;
+  }
+  return 0;
+}
+
+int mirror_remote_namespace_set(librados::IoCtx *ioctx,
+                                const std::string &mirror_namespace) {
+  bufferlist in_bl;
+  encode(mirror_namespace, in_bl);
+
+  bufferlist out_bl;
+  int r = ioctx->exec(RBD_MIRRORING, "rbd", "mirror_namespace_set", in_bl,
+                      out_bl);
+  if (r < 0) {
+    return r;
+  }
+  return 0;
+}
+
 void mirror_peer_list_start(librados::ObjectReadOperation *op) {
   bufferlist bl;
   op->exec("rbd", "mirror_peer_list", bl);

diff --git a/src/cls/rbd/cls_rbd_client.h b/src/cls/rbd/cls_rbd_client.h
@@ -389,6 +389,14 @@ int mirror_mode_get(librados::IoCtx *ioctx,
 int mirror_mode_set(librados::IoCtx *ioctx,
                     cls::rbd::MirrorMode mirror_mode);
 
+void mirror_remote_namespace_get_start(librados::ObjectReadOperation *op);
+int mirror_remote_namespace_get_finish(ceph::buffer::list::const_iterator *it,
+                                       std::string *mirror_namespace);
+int mirror_remote_namespace_get(librados::IoCtx *ioctx,
+				std::string *mirror_namespace);
+int mirror_remote_namespace_set(librados::IoCtx *ioctx,
+				const std::string &mirror_namespace);
+
 int mirror_peer_ping(librados::IoCtx *ioctx,
                      const std::string& site_name,
                      const std::string& fsid);

diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h
@@ -577,6 +577,11 @@ CEPH_RBD_API int rbd_mirror_mode_get(rados_ioctx_t io_ctx,
                                      rbd_mirror_mode_t *mirror_mode);
 CEPH_RBD_API int rbd_mirror_mode_set(rados_ioctx_t io_ctx,
                                      rbd_mirror_mode_t mirror_mode);
+CEPH_RBD_API int rbd_mirror_remote_namespace_get(rados_ioctx_t io_ctx,
+                                                 char *remote_namespace,
+				                 size_t *max_len);
+CEPH_RBD_API int rbd_mirror_remote_namespace_set(rados_ioctx_t io_ctx,
+                                                 const char *remote_namespace);
 
 CEPH_RBD_API int rbd_mirror_uuid_get(rados_ioctx_t io_ctx,
                                      char *uuid, size_t *max_len);

diff --git a/src/include/rbd/librbd.hpp b/src/include/rbd/librbd.hpp
@@ -357,6 +357,10 @@ class CEPH_RBD_API RBD
   int mirror_mode_get(IoCtx& io_ctx, rbd_mirror_mode_t *mirror_mode);
   int mirror_mode_set(IoCtx& io_ctx, rbd_mirror_mode_t mirror_mode);
 
+  int mirror_remote_namespace_get(IoCtx& io_ctx,
+                                  std::string* remote_namespace);
+  int mirror_remote_namespace_set(IoCtx& io_ctx,
+                                  const std::string& remote_namespace);
   int mirror_uuid_get(IoCtx& io_ctx, std::string* mirror_uuid);
 
   int mirror_peer_bootstrap_create(IoCtx& io_ctx, std::string* token);