Feature better auto crop (#48)

* switch auto crop code to right-open intervals

so that size is computed correctly

* better auto crop algorithm

expands left/right based on area increase

* improve algorithm with multiple start locations, add real test case

* keep pano selected on full-res failure

* add crop shadow

* tidy

Author: krupkat

tests/CMakeLists.txt

@@ -18,6 +18,8 @@ target_include_directories(AutoCropTest PRIVATE
   ".."
 )
 
+copy_file(AutoCropTest ${CMAKE_CURRENT_SOURCE_DIR}/data/mask.png)
+
 add_executable(StitcherTest 
   stitcher_pipeline_test.cc
   ../xpano/algorithm/algorithm.cc

tests/auto_crop_test.cc

@@ -2,6 +2,7 @@
 
 #include <catch2/catch_test_macros.hpp>
 #include <opencv2/core.hpp>
+#include <opencv2/imgcodecs.hpp>
 
 #include "xpano/utils/vec.h"
 
@@ -22,47 +23,47 @@ TEST_CASE("Auto crop full mask / even size") {
   auto result = FindLargestCrop(mask);
   REQUIRE(result.has_value());
   CHECK(result->start == Point2i{0, 0});
-  CHECK(result->end == Point2i{19, 9});
+  CHECK(result->end == Point2i{20, 10});
 }
 
 TEST_CASE("Auto crop full mask / odd size") {
   cv::Mat mask(10, 21, CV_8U, cv::Scalar(kMaskValueOn));
   auto result = FindLargestCrop(mask);
   REQUIRE(result.has_value());
   CHECK(result->start == Point2i{0, 0});
-  CHECK(result->end == Point2i{20, 9});
+  CHECK(result->end == Point2i{21, 10});
 }
 
 TEST_CASE("Auto crop single column mask") {
   cv::Mat mask(10, 1, CV_8U, cv::Scalar(kMaskValueOn));
   auto result = FindLargestCrop(mask);
   REQUIRE(result.has_value());
   CHECK(result->start == Point2i{0, 0});
-  CHECK(result->end == Point2i{0, 9});
+  CHECK(result->end == Point2i{1, 10});
 }
 
 TEST_CASE("Auto crop two columns mask") {
   cv::Mat mask(10, 2, CV_8U, cv::Scalar(kMaskValueOn));
   auto result = FindLargestCrop(mask);
   REQUIRE(result.has_value());
   CHECK(result->start == Point2i{0, 0});
-  CHECK(result->end == Point2i{1, 9});
+  CHECK(result->end == Point2i{2, 10});
 }
 
 TEST_CASE("Auto crop single row mask") {
   cv::Mat mask(1, 20, CV_8U, cv::Scalar(kMaskValueOn));
   auto result = FindLargestCrop(mask);
   REQUIRE(result.has_value());
   CHECK(result->start == Point2i{0, 0});
-  CHECK(result->end == Point2i{19, 0});
+  CHECK(result->end == Point2i{20, 1});
 }
 
 TEST_CASE("Auto crop two rows mask") {
   cv::Mat mask(2, 20, CV_8U, cv::Scalar(kMaskValueOn));
   auto result = FindLargestCrop(mask);
   REQUIRE(result.has_value());
   CHECK(result->start == Point2i{0, 0});
-  CHECK(result->end == Point2i{19, 1});
+  CHECK(result->end == Point2i{20, 2});
 }
 
 TEST_CASE("Auto crop mask with rows set") {
@@ -73,7 +74,7 @@ TEST_CASE("Auto crop mask with rows set") {
     auto result = FindLargestCrop(mask);
     REQUIRE(result.has_value());
     CHECK(result->start == Point2i{0, 5});
-    CHECK(result->end == Point2i{19, 5});
+    CHECK(result->end == Point2i{20, 6});
   }
 
   SECTION("two rows") {
@@ -82,7 +83,7 @@ TEST_CASE("Auto crop mask with rows set") {
     auto result = FindLargestCrop(mask);
     REQUIRE(result.has_value());
     CHECK(result->start == Point2i{0, 5});
-    CHECK(result->end == Point2i{19, 6});
+    CHECK(result->end == Point2i{20, 7});
   }
 }
 
@@ -91,10 +92,8 @@ TEST_CASE("Auto crop mask with empty column") {
   mask.col(5) = 0;
   auto result = FindLargestCrop(mask);
   REQUIRE(result.has_value());
-  // Algorithm will stop when encountering empty column 5
-  // this is to simplify the implementation
   CHECK(result->start == Point2i{6, 0});
-  CHECK(result->end == Point2i{13, 9});
+  CHECK(result->end == Point2i{20, 10});
 }
 
 TEST_CASE("Auto crop empty matrix") {
@@ -114,7 +113,7 @@ II->1 1 1 1 1 1
     1 1 0 1 1 1
             IV
 */
-TEST_CASE("Auto crop complex case") {
+TEST_CASE("Auto crop complex case I") {
   cv::Mat mask(6, 6, CV_8U, cv::Scalar(kMaskValueOn));
   mask.at<unsigned char>(0, 4) = 0;
   mask.at<unsigned char>(1, 2) = 0;
@@ -126,7 +125,35 @@ TEST_CASE("Auto crop complex case") {
   auto result = FindLargestCrop(mask);
   REQUIRE(result.has_value());
   CHECK(result->start == Point2i{1, 2});
-  CHECK(result->end == Point2i{4, 4});
+  CHECK(result->end == Point2i{5, 5});
+}
+
+/*        X
+    1 1 1 1 1 1 X
+    1 1 1 1 1 1
+    1 1 0 1 1 1
+    0 1 1 1 1 1
+    1 1 1 1 1 1
+    1 1 1 1 1 1 X
+              X
+*/
+TEST_CASE("Auto crop complex case II") {
+  cv::Mat mask(6, 6, CV_8U, cv::Scalar(kMaskValueOn));
+  mask.at<unsigned char>(2, 2) = 0;
+  mask.at<unsigned char>(3, 0) = 0;
+
+  auto result = FindLargestCrop(mask);
+  REQUIRE(result.has_value());
+  CHECK(result->start == Point2i{3, 0});
+  CHECK(result->end == Point2i{6, 6});
+}
+
+TEST_CASE("Real life example") {
+  auto mask = cv::imread("mask.png", cv::IMREAD_UNCHANGED);
+  auto result = FindLargestCrop(mask);
+  REQUIRE(result.has_value());
+  CHECK(result->start == Point2i{67, 659});
+  CHECK(result->end == Point2i{5985, 2950});
 }
 
 // NOLINTEND(readability-magic-numbers)

tests/data/mask.png

@@ -5,8 +5,8 @@
 #include <vector>
 
 #include <opencv2/core.hpp>
-#include <spdlog/spdlog.h>
 
+#include "xpano/constants.h"
 #include "xpano/utils/rect.h"
 #include "xpano/utils/vec.h"
 
@@ -18,6 +18,8 @@ struct Line {
   int end;
 };
 
+bool IsLineValid(const Line& line) { return line.start < line.end; }
+
 int Length(const Line& line) { return line.end - line.start; }
 
 bool IsSet(unsigned char value) { return value == kMaskValueOn; }
@@ -27,19 +29,19 @@ std::optional<Line> FindLongestLineInColumn(cv::Mat column) {
   std::optional<Line> longest_line;
 
   if (IsSet(column.at<unsigned char>(0, 0))) {
-    current_line = Line{0, 0};
+    current_line = Line{0, 1};
   }
 
   for (int i = 1; i < column.rows; i++) {
     auto prev = IsSet(column.at<unsigned char>(i - 1, 0));
     auto current = IsSet(column.at<unsigned char>(i, 0));
 
     if (!prev && current) {
-      current_line = Line{i, i};
+      current_line = Line{i, i + 1};
     }
 
     if (prev && current) {
-      current_line->end = i;
+      current_line->end = i + 1;
     }
 
     if (prev && !current) {
@@ -59,62 +61,77 @@ std::optional<Line> FindLongestLineInColumn(cv::Mat column) {
   return longest_line;
 }
 
+std::optional<utils::RectPPi> FindLargestCrop(const std::vector<Line>& lines,
+                                              const Line& invalid_line,
+                                              int seed) {
+  if (!IsLineValid(lines[seed])) {
+    return {};
+  }
+  auto current_rect =
+      utils::RectPPi{{seed, lines[seed].start}, {seed + 1, lines[seed].end}};
+  auto largest_rect = current_rect;
+
+  if (lines.size() == 1) {
+    return largest_rect;
+  }
+
+  int left = seed - 1;
+  int right = seed + static_cast<int>(lines.size() % 2);
+  auto left_line = lines[left];
+  auto right_line = lines[right];
+
+  while (IsLineValid(left_line) || IsLineValid(right_line)) {
+    auto left_rect = utils::RectPPi{
+        {left, std::max(left_line.start, current_rect.start[1])},
+        {current_rect.end[0], std::min(left_line.end, current_rect.end[1])}};
+
+    auto right_rect = utils::RectPPi{
+        {current_rect.start[0],
+         std::max(right_line.start, current_rect.start[1])},
+        {right + 1, std::min(right_line.end, current_rect.end[1])}};
+
+    if (utils::Area(left_rect) > utils::Area(right_rect)) {
+      current_rect = left_rect;
+      left_line = (left == 0) ? invalid_line : lines[--left];
+    } else {
+      current_rect = right_rect;
+      right_line = (right == lines.size() - 1) ? invalid_line : lines[++right];
+    }
+
+    if (utils::Area(current_rect) >= utils::Area(largest_rect)) {
+      largest_rect = current_rect;
+    }
+  }
+
+  return largest_rect;
+}
+
 }  // namespace
 
 // Approximage solution only.
 // Full solution would be https://stackoverflow.com/questions/2478447
-// This algorithm starts in the middle and expands the rectangle simultaneously
-// to the left and right. For the sake of simplicity if an empty column is
-// encountered, returns the current largest rectangle.
+// This algorithm starts in multiple sampled locations and expands
+// the rectangles in the direction with the larger area.
 std::optional<utils::RectPPi> FindLargestCrop(const cv::Mat& mask) {
+  if (mask.empty()) {
+    return {};
+  }
   Line invalid_line = {mask.rows, 0};
   std::vector<Line> lines(mask.cols);
   for (int i = 0; i < mask.cols; i++) {
     auto longest_line = FindLongestLineInColumn(mask.col(i));
     lines[i] = longest_line.value_or(invalid_line);
   }
 
-  int half_size = mask.cols / 2;
-  auto is_line_valid = [](const Line& line) { return line.start <= line.end; };
-  if (mask.cols == 0 || !is_line_valid(lines[half_size])) {
-    return {};
-  }
-  auto current_rect = utils::RectPPi{{half_size, lines[half_size].start},
-                                     {half_size, lines[half_size].end}};
-  auto largest_rect = current_rect;
+  std::optional<utils::RectPPi> largest_rect;
 
-  int left_start = half_size - 1;
-  int right_start = half_size + mask.cols % 2;
-  for (int i = 0; i < half_size; i++) {
-    auto left = left_start - i;
-    auto right = right_start + i;
-    auto left_line = lines[left];
-    auto right_line = lines[right];
-
-    if (!is_line_valid(left_line)) {
-      spdlog::warn("Auto crop: empty panorama at x = {}", left);
-      return largest_rect;
-    }
+  int num_samples = 1 + mask.cols / kAutoCropSamplingDistance;
+  for (int i = 0; i < num_samples; i++) {
+    int start = (i + 1) * mask.cols / (num_samples + 1);
+    auto current_rect = FindLargestCrop(lines, invalid_line, start);
 
-    if (!is_line_valid(right_line)) {
-      spdlog::warn("Auto crop: empty panorama at x = {}", right);
-      return largest_rect;
-    }
-
-    current_rect.start[0] = left;
-    current_rect.end[0] = right;
-
-    if (int top = std::max(left_line.start, right_line.start);
-        top > current_rect.start[1]) {
-      current_rect.start[1] = top;
-    }
-
-    if (int bottom = std::min(left_line.end, right_line.end);
-        bottom < current_rect.end[1]) {
-      current_rect.end[1] = bottom;
-    }
-
-    if (utils::Area(current_rect) >= utils::Area(largest_rect)) {
+    if (current_rect && (!largest_rect || utils::Area(*current_rect) >=
+                                              utils::Area(*largest_rect))) {
       largest_rect = current_rect;
     }
   }

xpano/algorithm/auto_crop.cc

@@ -69,5 +69,6 @@ constexpr float kDefaultPaniniB = 1.0f;
 const std::string kConfigFilename = "config.txt";
 
 constexpr int kCropEdgeTolerance = 10;
+constexpr int kAutoCropSamplingDistance = 512;
 
 }  // namespace xpano

xpano/constants.h

@@ -39,6 +39,21 @@ auto CropRectPP(const utils::RectPVf& image, const utils::RectRRf& crop_rect) {
 
 void Overlay(const utils::RectRRf& crop_rect, const utils::RectPVf& image) {
   auto rect = CropRectPP(image, crop_rect);
+
+  const auto transparent_color = ImColor(0, 0, 0, 128);
+  auto start = ImVec2(image.start[0], image.start[1]);
+  auto end = ImVec2(rect.start[0], image.start[1] + image.size[1]);
+  ImGui::GetWindowDrawList()->AddRectFilled(start, end, transparent_color);
+  start = ImVec2(rect.end[0], image.start[1]);
+  end = ImVec2(image.start[0] + image.size[0], image.start[1] + image.size[1]);
+  ImGui::GetWindowDrawList()->AddRectFilled(start, end, transparent_color);
+  start = ImVec2(rect.start[0], image.start[1]);
+  end = ImVec2(rect.end[0], rect.start[1]);
+  ImGui::GetWindowDrawList()->AddRectFilled(start, end, transparent_color);
+  start = ImVec2(rect.start[0], rect.end[1]);
+  end = ImVec2(rect.end[0], image.start[1] + image.size[1]);
+  ImGui::GetWindowDrawList()->AddRectFilled(start, end, transparent_color);
+
   const auto crop_color = ImColor(255, 255, 255, 255);
   ImGui::GetWindowDrawList()->AddRect(utils::ImVec(rect.start),
                                       utils::ImVec(rect.end), crop_color, 0.0f,

xpano/gui/panels/preview_pane.cc

@@ -162,7 +162,9 @@ auto ResolveStitchingResultFuture(
     *status_message = {fmt::format("Failed to stitch pano {}", result.pano_id),
                        algorithm::ToString(result.status)};
     spdlog::info(*status_message);
-    plot_pane->Reset();
+    if (!result.full_res) {
+      plot_pane->Reset();
+    }
     return {};
   }
 

xpano/gui/pano_gui.cc

@@ -118,7 +118,11 @@ StitchingResult StitcherPipeline::RunStitchingPipeline(
   progress_.NotifyTaskDone();
 
   if (status != cv::Stitcher::OK) {
-    return StitchingResult{.pano_id = options.pano_id, .status = status};
+    return StitchingResult{
+        .pano_id = options.pano_id,
+        .full_res = options.full_res,
+        .status = status,
+    };
   }
 
   std::optional<utils::RectRRf> auto_crop;