From 8127e3811d2cfda0d286accdeb598c1b5594aac9 Mon Sep 17 00:00:00 2001 From: Marek Simonik Date: Thu, 17 Sep 2020 12:27:55 +0200 Subject: [PATCH] Introduced support for LiDAR depth streaming. Use with the iOS app Record3D 1.4 and newer. --- README.md | 4 +++- demo-main.py | 7 +++++-- include/record3d/Record3DStream.h | 32 +++++++++++++++++++++---------- setup.py | 2 +- src/DemoMain.cpp | 32 +++++++++++++++++++++++++------ src/Record3DStream.cpp | 13 +++++++++---- 6 files changed, 66 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 1464724..8c4f916 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # [*Record3D – Point Cloud Animation and Streaming*](https://record3d.app/): the accompanying library +**2020/09/17 Update**: Introduced LiDAR support. To be used with Record3D 1.4 and newer. + This project provides C++ and Python libraries for the [iOS Record3D app](https://record3d.app/) which allows you (among other features) to live-stream RGB**D** video from iOS devices with TrueDepth camera to a computer via USB cable. @@ -47,7 +49,7 @@ After running the following, you will find compiled static library in the `build ## Sample applications There is a Python (`demo-main.py`) and C++ (`src/DemoMain.cpp`) sample project that demonstrates how to use the library to receive and display RGBD stream. -Before running the sample applications, connect your iOS device to your computer and open the Record3D iOS app. Go to the Settings tab and enable "USB Streaming mode". Note that the fr +Before running the sample applications, connect your iOS device to your computer and open the Record3D iOS app. Go to the Settings tab and enable "USB Streaming mode". ### Python After installing the `record3d` library, run `python demo-main.py` and press the record button to start streaming RGBD data. diff --git a/demo-main.py b/demo-main.py index 780ee25..591272c 100644 --- a/demo-main.py +++ b/demo-main.py @@ -52,8 +52,11 @@ def start_processing_stream(self): # You can now e.g. create point cloud by projecting the depth map using the intrinsic matrix. # Postprocess it - depth = cv2.flip(depth, 1) - rgb = cv2.flip(rgb, 1) + are_truedepth_camera_data_being_streamed = depth.shape[0] == 640 + if are_truedepth_camera_data_being_streamed: + depth = cv2.flip(depth, 1) + rgb = cv2.flip(rgb, 1) + rgb = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR) # Show the RGBD Stream diff --git a/include/record3d/Record3DStream.h b/include/record3d/Record3DStream.h index 59aef57..b19b3df 100644 --- a/include/record3d/Record3DStream.h +++ b/include/record3d/Record3DStream.h @@ -94,8 +94,8 @@ namespace Record3D public: // Constants static constexpr uint16_t DEVICE_PORT{ 1337 }; /** Port on iDevice that we are listening to for RGBD stream. */ - static constexpr uint32_t FRAME_WIDTH{ 480 }; /** Width of the RGB and Depth components of the RGBD stream. */ - static constexpr uint32_t FRAME_HEIGHT{ 640 }; /** Height of the RGB and Depth components of the RGBD stream. */ + static constexpr uint32_t MAXIMUM_FRAME_WIDTH{ 480 }; /** Maximum width of the RGB and Depth components of the RGBD stream. */ + static constexpr uint32_t MAXIMUM_FRAME_HEIGHT{ 640 }; /** Maximum height of the RGB and Depth components of the RGBD stream. */ #ifdef PYTHON_BINDINGS_BUILD /** @@ -141,12 +141,16 @@ namespace Record3D */ py::array_t GetCurrentDepthFrame() { - auto result = py::array_t(FRAME_WIDTH * FRAME_HEIGHT); + size_t currentFrameWidth = currentFrameWidth_; + size_t currentFrameHeight = currentFrameHeight_; + + size_t bufferSize = currentFrameWidth * currentFrameHeight * sizeof(float); + auto result = py::array_t(currentFrameWidth * currentFrameHeight); auto result_buffer = result.request(); float *result_ptr = (float *) result_buffer.ptr; - std::memcpy(result_ptr, depthImageBuffer_.data(), depthImageBuffer_.size()); - result.resize(std::vector{static_cast(FRAME_HEIGHT), static_cast(FRAME_WIDTH)}); + std::memcpy(result_ptr, depthImageBuffer_.data(), bufferSize); + result.resize(std::vector{static_cast(currentFrameHeight), static_cast(currentFrameWidth)}); return result; } @@ -158,12 +162,17 @@ namespace Record3D */ py::array_t GetCurrentRGBFrame() { - auto result = py::array_t(FRAME_WIDTH * FRAME_HEIGHT * 3); + size_t currentFrameWidth = currentFrameWidth_; + size_t currentFrameHeight = currentFrameHeight_; + + constexpr int numChannels = 3; + size_t bufferSize = currentFrameWidth * currentFrameHeight * numChannels * sizeof(uint8_t); + auto result = py::array_t(bufferSize); auto result_buffer = result.request(); - float *result_ptr = (float *) result_buffer.ptr; + uint8_t *result_ptr = (uint8_t *) result_buffer.ptr; - std::memcpy(result_ptr, RGBImageBuffer_.data(), RGBImageBuffer_.size()); - result.resize(std::vector{static_cast(FRAME_HEIGHT), static_cast(FRAME_WIDTH), 3}); + std::memcpy(result_ptr, RGBImageBuffer_.data(), bufferSize); + result.resize(std::vector{static_cast(currentFrameHeight), static_cast(currentFrameWidth), numChannels}); return result; } @@ -179,7 +188,10 @@ namespace Record3D } #endif private: - static constexpr size_t depthBufferSize_{ FRAME_WIDTH * FRAME_HEIGHT * sizeof( float ) }; /** Size in bytes of decompressed Depth frame. */ + static constexpr size_t depthBufferSize_{MAXIMUM_FRAME_WIDTH * MAXIMUM_FRAME_HEIGHT * sizeof( float ) }; /** Size in bytes of decompressed Depth frame. */ + + size_t currentFrameWidth_{ MAXIMUM_FRAME_WIDTH }; + size_t currentFrameHeight_{ MAXIMUM_FRAME_HEIGHT }; uint8_t* compressedDepthBuffer_{ nullptr }; /** Preallocated buffer holding decompressed depth data. */ uint8_t* lzfseScratchBuffer_{ nullptr }; /** Preallocated LZFSE scratch buffer. */ diff --git a/setup.py b/setup.py index 28d1616..a6fe392 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ def build_extension(self, ext): setup( name='record3d', - version='1.0.0', + version='1.2.0', license='lgpl-2.1', author='Marek Simonik', author_email='admin@record3d.app', diff --git a/src/DemoMain.cpp b/src/DemoMain.cpp index e83ae95..e42bb58 100644 --- a/src/DemoMain.cpp +++ b/src/DemoMain.cpp @@ -67,8 +67,14 @@ class Record3DDemoApp #ifdef HAS_OPENCV // Postprocess images cv::cvtColor( imgRGB_, imgRGB_, cv::COLOR_RGB2BGR ); - cv::flip( imgRGB_, imgRGB_, 1 ); - cv::flip( imgDepth_, imgDepth_, 1 ); + + // The TrueDepth camera is a selfie camera; we mirror the RGBD frame so it looks plausible. + bool areTrueDepthDataBeingStreamed = imgDepth_.rows == Record3D::Record3DStream::MAXIMUM_FRAME_HEIGHT && imgDepth_.cols == Record3D::Record3DStream::MAXIMUM_FRAME_WIDTH; + if ( areTrueDepthDataBeingStreamed ) + { + cv::flip( imgRGB_, imgRGB_, 1 ); + cv::flip( imgDepth_, imgDepth_, 1 ); + } // Show images cv::imshow( "RGB", imgRGB_ ); @@ -97,8 +103,22 @@ class Record3DDemoApp Record3D::IntrinsicMatrixCoeffs $K) { #ifdef HAS_OPENCV - memcpy( imgRGB_.data, $rgbFrame.data(), $rgbFrame.size()); - memcpy( imgDepth_.data, $depthFrame.data(), $depthFrame.size()); + // When we switch between the TrueDepth and the LiDAR camera, the size frame size changes. + // Recreate the RGB and Depth images with fitting size. + if ( imgRGB_.rows != $frameHeight || imgRGB_.cols != $frameWidth + || imgDepth_.rows != $frameHeight || imgDepth_.cols != $frameWidth ) + { + imgRGB_.release(); + imgDepth_.release(); + + imgRGB_ = cv::Mat::zeros( $frameHeight, $frameWidth, CV_8UC3); + imgDepth_ = cv::Mat::zeros( $frameHeight, $frameWidth, CV_32F ); + } + + // The `BufferRGB` and `BufferDepth` may be larger than the actual payload, therefore the true frame size is computed. + constexpr int numRGBChannels = 3; + memcpy( imgRGB_.data, $rgbFrame.data(), $frameWidth * $frameHeight * numRGBChannels * sizeof(uint8_t)); + memcpy( imgDepth_.data, $depthFrame.data(), $frameWidth * $frameHeight * sizeof(float)); #endif mainThreadLock_.unlock(); } @@ -111,8 +131,8 @@ class Record3DDemoApp #endif #ifdef HAS_OPENCV - cv::Mat imgRGB_ = cv::Mat::zeros( Record3D::Record3DStream::FRAME_HEIGHT, Record3D::Record3DStream::FRAME_WIDTH, CV_8UC3); - cv::Mat imgDepth_ = cv::Mat::zeros( Record3D::Record3DStream::FRAME_HEIGHT, Record3D::Record3DStream::FRAME_WIDTH, CV_32F ); + cv::Mat imgRGB_ = cv::Mat::zeros(Record3D::Record3DStream::MAXIMUM_FRAME_HEIGHT, Record3D::Record3DStream::MAXIMUM_FRAME_WIDTH, CV_8UC3);; + cv::Mat imgDepth_ = cv::Mat::zeros(Record3D::Record3DStream::MAXIMUM_FRAME_HEIGHT, Record3D::Record3DStream::MAXIMUM_FRAME_WIDTH, CV_32F );; #endif }; diff --git a/src/Record3DStream.cpp b/src/Record3DStream.cpp index b9167da..0667a41 100644 --- a/src/Record3DStream.cpp +++ b/src/Record3DStream.cpp @@ -20,7 +20,7 @@ namespace Record3D depthImageBuffer_.resize( depthBufferSize_ ); constexpr int numRGBChannels = 3; - RGBImageBuffer_.resize( Record3DStream::FRAME_WIDTH * Record3DStream::FRAME_HEIGHT * sizeof( uint8_t ) * numRGBChannels ); + RGBImageBuffer_.resize(Record3DStream::MAXIMUM_FRAME_WIDTH * Record3DStream::MAXIMUM_FRAME_HEIGHT * sizeof( uint8_t ) * numRGBChannels ); } Record3DStream::~Record3DStream() @@ -107,6 +107,8 @@ namespace Record3D struct Record3DHeader { + uint32_t frameWidth; + uint32_t frameHeight; uint32_t rgbSize; uint32_t depthSize; }; @@ -153,7 +155,7 @@ namespace Record3D currSize = record3DHeader.rgbSize; int loadedWidth, loadedHeight, loadedChannels; uint8_t* rgbPixels = stbi_load_from_memory( rawMessageBuffer.data() + offset, currSize, &loadedWidth, &loadedHeight, &loadedChannels, STBI_rgb ); - memcpy( RGBImageBuffer_.data(), rgbPixels, RGBImageBuffer_.size()); + memcpy( RGBImageBuffer_.data(), rgbPixels, loadedWidth * loadedHeight * loadedChannels * sizeof(uint8_t)); stbi_image_free( rgbPixels ); offset += currSize; @@ -163,10 +165,13 @@ namespace Record3D if ( onNewFrame ) { + currentFrameWidth_ = record3DHeader.frameWidth; + currentFrameHeight_ = record3DHeader.frameHeight; + #ifdef PYTHON_BINDINGS_BUILD - onNewFrame(); + onNewFrame( ); #else - onNewFrame( RGBImageBuffer_, depthImageBuffer_, FRAME_WIDTH, FRAME_HEIGHT, intrinsicMatrixCoeffs_ ); + onNewFrame( RGBImageBuffer_, depthImageBuffer_, record3DHeader.frameWidth, record3DHeader.frameHeight, intrinsicMatrixCoeffs_ ); #endif } }