librbd/migration: add nbd stream

Co-authored-by: Ilya Dryomov <[email protected]>
Signed-off-by: Effi Ofer <[email protected]>
Signed-off-by: Ilya Dryomov <[email protected]>

Author: effi-ofer

CMakeLists.txt

@@ -259,6 +259,12 @@ if(WITH_RBD AND LINUX)
   set(HAVE_LIBCRYPTSETUP ${LIBCRYPTSETUP_FOUND})
 endif()
 
+# libnbd
+if(WITH_RBD AND NOT WIN32)
+  find_package(libnbd 1.0 REQUIRED)
+  set(HAVE_LIBNBD ${LIBNBD_FOUND})
+endif()
+
 include(CMakeDependentOption)
 
 CMAKE_DEPENDENT_OPTION(WITH_LIBURING "Enable io_uring bluestore backend" ON

ceph.spec.in

@@ -286,6 +286,7 @@ BuildRequires:	gperftools-devel >= 2.4
 BuildRequires:	libaio-devel
 BuildRequires:	libblkid-devel >= 2.17
 BuildRequires:	cryptsetup-devel
+BuildRequires:	libnbd-devel
 BuildRequires:	libcurl-devel
 BuildRequires:	libcap-devel
 BuildRequires:	libcap-ng-devel

cmake/modules/Findlibnbd.cmake

@@ -0,0 +1,33 @@
+# - Find libnbd
+# Sets the following:
+#
+# LIBNBD_INCLUDE_DIR
+# LIBNBD_LIBRARIES
+# LIBNBD_VERSION
+# LIBNBD_FOUND
+
+find_package(PkgConfig QUIET REQUIRED)
+pkg_search_module(PC_libnbd libnbd)
+
+find_path(LIBNBD_INCLUDE_DIR
+        NAMES libnbd.h
+        PATHS ${PC_libnbd_INCLUDE_DIRS})
+
+find_library(LIBNBD_LIBRARIES
+        NAMES libnbd.so
+        PATHS ${PC_libnbd_LIBRARY_DIRS})
+
+set(LIBNBD_VERSION ${PC_libnbd_VERSION})
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(libnbd
+        REQUIRED_VARS
+        LIBNBD_INCLUDE_DIR
+        LIBNBD_LIBRARIES
+        VERSION_VAR LIBNBD_VERSION)
+
+mark_as_advanced(
+  LIBNBD_LIBRARIES
+  LIBNBD_INCLUDE_DIR
+  LIBNBD_VERSION)

debian/control

@@ -54,6 +54,7 @@ Build-Depends: automake,
                liblttng-ust-dev,
                liblua5.3-dev,
                liblz4-dev (>= 0.0~r131),
+               libnbd-dev,
                libncurses-dev,
                libnss3-dev,
                liboath-dev,

doc/rbd/rbd-live-migration.rst

@@ -20,7 +20,7 @@ parent.
 The live-migration process can also be used in an import-only mode where the
 source image remains unmodified and the target image can be linked to an image
 in another Ceph cluster or to an external data source such as a backing file,
-HTTP(s) file, or S3 object.
+HTTP(s) file, S3 object, or NBD export.
 
 The live-migration copy process can safely run in the background while the new
 target image is in use. There is currently a requirement to temporarily stop
@@ -145,8 +145,8 @@ The general format for the ``source-spec`` JSON is as follows::
         }
 
 The following formats are currently supported: ``native``, ``qcow``, and
-``raw``. The following streams are currently supported: ``file``, ``http``, and
-``s3``.
+``raw``. The following streams are currently supported: ``file``, ``http``,
+``s3``, and ``nbd``.
 
 Formats
 ~~~~~~~
@@ -306,6 +306,19 @@ as follows::
   stored in the config-key store via ``ceph config-key set <key-path> <value>``
   (e.g. ``ceph config-key set rbd/s3/access_key NX5QOQKC6BH2IDN8HC7A``).
 
+The ``nbd`` stream can be used to import from a remote NBD export. Its
+``source-spec`` JSON is encoded as follows::
+
+        {
+            <format unique parameters>
+            "stream": {
+                "type": "nbd",
+                "server": "<server>",
+                "port": "<port>"
+            }
+        }
+
+
 Execute Migration
 =================
 

qa/workunits/rbd/cli_migration.sh

@@ -8,6 +8,7 @@ IMAGE3=image3
 IMAGES="${IMAGE1} ${IMAGE2} ${IMAGE3}"
 
 cleanup() {
+    kill_nbd_server
     cleanup_tempdir
     remove_images
 }
@@ -65,6 +66,10 @@ remove_images() {
     done
 }
 
+kill_nbd_server() {
+    pkill -9 qemu-nbd || true
+}
+
 show_diff()
 {
     local file1=$1
@@ -390,6 +395,42 @@ EOF
     remove_image "${dest_image}"
 }
 
+test_import_nbd_stream() {
+    local base_image=$1
+    local dest_image=$2
+
+    qemu-nbd -f qcow2 --read-only --shared 10 --persistent --fork \
+        ${TEMPDIR}/${base_image}.qcow2
+
+    cat > ${TEMPDIR}/spec.json <<EOF
+{
+  "type": "raw",
+  "stream": {
+    "type": "nbd",
+    "server": "localhost",
+    "port": "10809"
+  }
+}
+EOF
+    cat ${TEMPDIR}/spec.json
+
+    cat ${TEMPDIR}/spec.json | rbd migration prepare --import-only \
+        --source-spec-path - ${dest_image}
+    compare_images ${base_image} ${dest_image}
+    rbd migration abort ${dest_image}
+
+    rbd migration prepare --import-only \
+        --source-spec-path ${TEMPDIR}/spec.json ${dest_image}
+    compare_images ${base_image} ${dest_image}
+    rbd migration execute ${dest_image}
+    compare_images ${base_image} ${dest_image}
+    rbd migration commit ${dest_image}
+    compare_images ${base_image} ${dest_image}
+    remove_image "${dest_image}"
+
+    kill_nbd_server
+}
+
 # make sure rbd pool is EMPTY.. this is a test script!!
 rbd ls 2>&1 | wc -l | grep -v '^0$' && echo "nonempty rbd pool, aborting!  run this script on an empty test cluster only." && exit 1
 
@@ -401,7 +442,10 @@ export_base_image ${IMAGE1}
 
 test_import_native_format ${IMAGE1} ${IMAGE2}
 test_import_qcow_format ${IMAGE1} ${IMAGE2}
+
 test_import_qcow2_format ${IMAGE2} ${IMAGE3}
+test_import_nbd_stream ${IMAGE2} ${IMAGE3}
+
 test_import_raw_format ${IMAGE1} ${IMAGE2}
 
 echo OK

src/include/config-h.in.cmake

@@ -393,6 +393,9 @@
 /* Define if libcryptsetup can be used (linux only) */
 #cmakedefine HAVE_LIBCRYPTSETUP
 
+/* Define if libnbd can be used */
+#cmakedefine HAVE_LIBNBD
+
 /* Shared library extension, such as .so, .dll or .dylib */
 #cmakedefine CMAKE_SHARED_LIBRARY_SUFFIX "@CMAKE_SHARED_LIBRARY_SUFFIX@"
 

src/librbd/CMakeLists.txt

@@ -219,6 +219,11 @@ if(LINUX AND HAVE_LIBCRYPTSETUP)
           crypto/luks/Magic.cc)
 endif()
 
+if(HAVE_LIBNBD)
+  list(APPEND librbd_internal_srcs
+          migration/NBDStream.cc)
+endif()
+
 add_library(rbd_api STATIC librbd.cc)
 add_library(rbd_internal STATIC
   ${librbd_internal_srcs}
@@ -240,6 +245,10 @@ if(LINUX AND HAVE_LIBCRYPTSETUP)
   target_include_directories(rbd_internal PRIVATE ${LIBCRYPTSETUP_INCLUDE_DIR})
   target_link_libraries(rbd_internal PRIVATE ${LIBCRYPTSETUP_LIBRARIES})
 endif()
+if(HAVE_LIBNBD)
+  target_include_directories(rbd_internal PRIVATE ${LIBNBD_INCLUDE_DIR})
+  target_link_libraries(rbd_internal PRIVATE ${LIBNBD_LIBRARIES})
+endif()
 
 add_custom_target(librbd_plugins)
 set(librbd_plugins_dir ${CEPH_INSTALL_PKGLIBDIR}/librbd)

src/librbd/migration/FileStream.cc

@@ -226,6 +226,18 @@ void FileStream<I>::read(io::Extents&& byte_extents, bufferlist* data,
 
 #endif // BOOST_ASIO_HAS_POSIX_STREAM_DESCRIPTOR
 
+template <typename I>
+void FileStream<I>::list_sparse_extents(io::Extents&& byte_extents,
+                                        io::SparseExtents* sparse_extents,
+                                        Context* on_finish) {
+  // TODO: list sparse extents based on SEEK_HOLE/SEEK_DATA
+  for (auto [byte_offset, byte_length] : byte_extents) {
+    sparse_extents->insert(byte_offset, byte_length,
+                           {io::SPARSE_EXTENT_STATE_DATA, byte_length});
+  }
+  on_finish->complete(0);
+}
+
 } // namespace migration
 } // namespace librbd
 

src/librbd/migration/FileStream.h

@@ -44,6 +44,10 @@ class FileStream : public StreamInterface {
   void read(io::Extents&& byte_extents, bufferlist* data,
             Context* on_finish) override;
 
+  void list_sparse_extents(io::Extents&& byte_extents,
+                           io::SparseExtents* sparse_extents,
+                           Context* on_finish) override;
+
 private:
   CephContext* m_cct;
   std::shared_ptr<AsioEngine> m_asio_engine;

src/librbd/migration/HttpStream.cc

@@ -77,6 +77,18 @@ void HttpStream<I>::read(io::Extents&& byte_extents, bufferlist* data,
   m_http_client->read(std::move(byte_extents), data, on_finish);
 }
 
+template <typename I>
+void HttpStream<I>::list_sparse_extents(io::Extents&& byte_extents,
+                                        io::SparseExtents* sparse_extents,
+                                        Context* on_finish) {
+  // no sparseness information -- list the full range as DATA
+  for (auto [byte_offset, byte_length] : byte_extents) {
+    sparse_extents->insert(byte_offset, byte_length,
+                           {io::SPARSE_EXTENT_STATE_DATA, byte_length});
+  }
+  on_finish->complete(0);
+}
+
 } // namespace migration
 } // namespace librbd
 

src/librbd/migration/HttpStream.h

@@ -45,6 +45,10 @@ class HttpStream : public StreamInterface {
   void read(io::Extents&& byte_extents, bufferlist* data,
             Context* on_finish) override;
 
+  void list_sparse_extents(io::Extents&& byte_extents,
+                           io::SparseExtents* sparse_extents,
+                           Context* on_finish) override;
+
 private:
   using HttpResponse = boost::beast::http::response<
     boost::beast::http::string_body>;