Skip to content

Commit f0fd04c

Browse files
krupkatkrupkat
authored
Feature auto crop (#47)
* add auto crop computation approx algorithm * move crop code to auto_crop.cc * handle edge cases in auto crop * added auto crop test, fix auto crop of odd size * tidy + iwyu
1 parent c24bb30 commit f0fd04c

File tree

13 files changed

+374
-12
lines changed

13 files changed

+374
-12
lines changed

‎CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ set(IMGUI_SOURCES
3030
set(XPANO_SOURCES
3131
"xpano/main.cc"
3232
"xpano/algorithm/algorithm.cc"
33+
"xpano/algorithm/auto_crop.cc"
3334
"xpano/algorithm/image.cc"
3435
"xpano/log/logger.cc"
3536
"xpano/gui/backends/base.cc"

‎tests/CMakeLists.txt‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,24 @@ find_package(Catch2 REQUIRED)
44
include(Catch)
55
include("${CMAKE_SOURCE_DIR}/misc/cmake/utils.cmake")
66

7+
add_executable(AutoCropTest
8+
auto_crop_test.cc
9+
../xpano/algorithm/auto_crop.cc)
10+
11+
target_link_libraries(AutoCropTest
12+
Catch2::Catch2WithMain
13+
${OPENCV_TARGETS}
14+
spdlog::spdlog
15+
)
16+
17+
target_include_directories(AutoCropTest PRIVATE
18+
".."
19+
)
20+
721
add_executable(StitcherTest
822
stitcher_pipeline_test.cc
923
../xpano/algorithm/algorithm.cc
24+
../xpano/algorithm/auto_crop.cc
1025
../xpano/algorithm/image.cc
1126
../xpano/pipeline/stitcher_pipeline.cc
1227
../xpano/utils/disjoint_set.cc)
@@ -62,6 +77,7 @@ target_include_directories(DisjointSetTest PRIVATE
6277
)
6378

6479
set(ALL_TEST_TARGETS
80+
AutoCropTest
6581
DisjointSetTest
6682
RectTest
6783
StitcherTest

‎tests/auto_crop_test.cc‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
#include "xpano/algorithm/auto_crop.h"
2+
3+
#include <catch2/catch_test_macros.hpp>
4+
#include <opencv2/core.hpp>
5+
6+
#include "xpano/utils/vec.h"
7+
8+
using xpano::algorithm::crop::FindLargestCrop;
9+
using xpano::algorithm::crop::kMaskValueOn;
10+
using xpano::utils::Point2i;
11+
12+
// NOLINTBEGIN(readability-magic-numbers)
13+
14+
TEST_CASE("Auto crop empty mask") {
15+
cv::Mat mask(10, 20, CV_8U, cv::Scalar(0));
16+
auto result = FindLargestCrop(mask);
17+
CHECK(!result.has_value());
18+
}
19+
20+
TEST_CASE("Auto crop full mask / even size") {
21+
cv::Mat mask(10, 20, CV_8U, cv::Scalar(kMaskValueOn));
22+
auto result = FindLargestCrop(mask);
23+
REQUIRE(result.has_value());
24+
CHECK(result->start == Point2i{0, 0});
25+
CHECK(result->end == Point2i{19, 9});
26+
}
27+
28+
TEST_CASE("Auto crop full mask / odd size") {
29+
cv::Mat mask(10, 21, CV_8U, cv::Scalar(kMaskValueOn));
30+
auto result = FindLargestCrop(mask);
31+
REQUIRE(result.has_value());
32+
CHECK(result->start == Point2i{0, 0});
33+
CHECK(result->end == Point2i{20, 9});
34+
}
35+
36+
TEST_CASE("Auto crop single column mask") {
37+
cv::Mat mask(10, 1, CV_8U, cv::Scalar(kMaskValueOn));
38+
auto result = FindLargestCrop(mask);
39+
REQUIRE(result.has_value());
40+
CHECK(result->start == Point2i{0, 0});
41+
CHECK(result->end == Point2i{0, 9});
42+
}
43+
44+
TEST_CASE("Auto crop two columns mask") {
45+
cv::Mat mask(10, 2, CV_8U, cv::Scalar(kMaskValueOn));
46+
auto result = FindLargestCrop(mask);
47+
REQUIRE(result.has_value());
48+
CHECK(result->start == Point2i{0, 0});
49+
CHECK(result->end == Point2i{1, 9});
50+
}
51+
52+
TEST_CASE("Auto crop single row mask") {
53+
cv::Mat mask(1, 20, CV_8U, cv::Scalar(kMaskValueOn));
54+
auto result = FindLargestCrop(mask);
55+
REQUIRE(result.has_value());
56+
CHECK(result->start == Point2i{0, 0});
57+
CHECK(result->end == Point2i{19, 0});
58+
}
59+
60+
TEST_CASE("Auto crop two rows mask") {
61+
cv::Mat mask(2, 20, CV_8U, cv::Scalar(kMaskValueOn));
62+
auto result = FindLargestCrop(mask);
63+
REQUIRE(result.has_value());
64+
CHECK(result->start == Point2i{0, 0});
65+
CHECK(result->end == Point2i{19, 1});
66+
}
67+
68+
TEST_CASE("Auto crop mask with rows set") {
69+
cv::Mat mask(10, 20, CV_8U, cv::Scalar(0));
70+
71+
SECTION("single row") {
72+
mask.row(5) = kMaskValueOn;
73+
auto result = FindLargestCrop(mask);
74+
REQUIRE(result.has_value());
75+
CHECK(result->start == Point2i{0, 5});
76+
CHECK(result->end == Point2i{19, 5});
77+
}
78+
79+
SECTION("two rows") {
80+
mask.row(5) = kMaskValueOn;
81+
mask.row(6) = kMaskValueOn;
82+
auto result = FindLargestCrop(mask);
83+
REQUIRE(result.has_value());
84+
CHECK(result->start == Point2i{0, 5});
85+
CHECK(result->end == Point2i{19, 6});
86+
}
87+
}
88+
89+
TEST_CASE("Auto crop mask with empty column") {
90+
cv::Mat mask(10, 20, CV_8U, cv::Scalar(kMaskValueOn));
91+
mask.col(5) = 0;
92+
auto result = FindLargestCrop(mask);
93+
REQUIRE(result.has_value());
94+
// Algorithm will stop when encountering empty column 5
95+
// this is to simplify the implementation
96+
CHECK(result->start == Point2i{6, 0});
97+
CHECK(result->end == Point2i{13, 9});
98+
}
99+
100+
TEST_CASE("Auto crop empty matrix") {
101+
cv::Mat mask;
102+
auto result = FindLargestCrop(mask);
103+
REQUIRE(!result.has_value());
104+
}
105+
106+
// Example from https://stackoverflow.com/questions/2478447
107+
/*
108+
I
109+
1 1 1 1 0 1
110+
1 1 0 1 1 0
111+
II->1 1 1 1 1 1
112+
0 1 1 1 1 1
113+
1 1 1 1 1 0 <--IV
114+
1 1 0 1 1 1
115+
IV
116+
*/
117+
TEST_CASE("Auto crop complex case") {
118+
cv::Mat mask(6, 6, CV_8U, cv::Scalar(kMaskValueOn));
119+
mask.at<unsigned char>(0, 4) = 0;
120+
mask.at<unsigned char>(1, 2) = 0;
121+
mask.at<unsigned char>(1, 5) = 0;
122+
mask.at<unsigned char>(3, 0) = 0;
123+
mask.at<unsigned char>(4, 5) = 0;
124+
mask.at<unsigned char>(5, 2) = 0;
125+
126+
auto result = FindLargestCrop(mask);
127+
REQUIRE(result.has_value());
128+
CHECK(result->start == Point2i{1, 2});
129+
CHECK(result->end == Point2i{4, 4});
130+
}
131+
132+
// NOLINTEND(readability-magic-numbers)

‎tests/rect_test.cc‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,3 +49,11 @@ TEST_CASE("Rect Aspect") {
4949
CHECK(Aspect(rect_pv) == Approx(0.5f));
5050
CHECK(Aspect(rect_pp) == Approx(0.5f));
5151
}
52+
53+
TEST_CASE("Rect Area") {
54+
auto rect_pv = Rect(Point2f{1.0f, 2.0f}, Vec2f{1.0f, 2.0f});
55+
auto rect_pp = Rect(Point2f{1.0f, 2.0f}, Point2f{2.0f, 4.0f});
56+
57+
CHECK(Area(rect_pv) == Approx(2.0f));
58+
CHECK(Area(rect_pp) == Approx(2.0f));
59+
}

‎tests/vec_test.cc‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,4 +301,21 @@ TEST_CASE("Multiply type checks") {
301301
}
302302
}
303303

304+
TEST_CASE("Multiply elements") {
305+
Vec2f vec1 = {2.0f, 2.5f};
306+
Vec2i vec2 = {4, 8};
307+
308+
REQUIRE(xpano::utils::MultiplyElements(vec1) == Approx(5.0f));
309+
REQUIRE(xpano::utils::MultiplyElements(vec2) == 32);
310+
}
311+
312+
TEST_CASE("Equality") {
313+
Vec2f vec1 = {1.0f, 2.0f};
314+
Vec2f vec2 = {1.0f, 2.0f};
315+
Vec2f vec3 = {1.0f, 3.0f};
316+
317+
REQUIRE(vec1 == vec2);
318+
REQUIRE(vec1 != vec3);
319+
}
320+
304321
// NOLINTEND(readability-magic-numbers)

‎xpano/algorithm/algorithm.cc‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include <opencv2/features2d.hpp>
1515
#include <opencv2/stitching.hpp>
1616

17+
#include "xpano/algorithm/auto_crop.h"
1718
#include "xpano/algorithm/image.h"
1819
#include "xpano/utils/disjoint_set.h"
1920
#include "xpano/utils/rect.h"
@@ -262,15 +263,13 @@ const char* Label(ProjectionType projection_type) {
262263
}
263264
}
264265

265-
// NOLINTBEGIN(performance-unnecessary-value-param)
266-
267-
utils::RectRRf FindLargestCropRectangle(cv::Mat /*mask*/) {
268-
auto start = utils::Ratio2f{0.0f};
269-
auto end = utils::Ratio2f{1.0f};
270-
// find largest area with 0xFF
271-
return Rect(start, end);
266+
std::optional<utils::RectRRf> FindLargestCrop(const cv::Mat& mask) {
267+
std::optional<utils::RectPPi> largest_rect = crop::FindLargestCrop(mask);
268+
if (!largest_rect) {
269+
return {};
270+
}
271+
auto image_end = utils::Point2i{mask.cols, mask.rows};
272+
return Rect(largest_rect->start / image_end, largest_rect->end / image_end);
272273
}
273274

274-
// NOLINTEND(performance-unnecessary-value-param)
275-
276275
} // namespace xpano::algorithm

‎xpano/algorithm/algorithm.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#pragma once
22

33
#include <array>
4+
#include <optional>
45
#include <string>
56
#include <vector>
67

@@ -81,6 +82,6 @@ StitchResult Stitch(const std::vector<cv::Mat>& images, StitchOptions options);
8182

8283
std::string ToString(cv::Stitcher::Status& status);
8384

84-
utils::RectRRf FindLargestCropRectangle(cv::Mat mask);
85+
std::optional<utils::RectRRf> FindLargestCrop(const cv::Mat& mask);
8586

8687
} // namespace xpano::algorithm

‎xpano/algorithm/auto_crop.cc‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
#include "xpano/algorithm/auto_crop.h"
2+
3+
#include <algorithm>
4+
#include <optional>
5+
#include <vector>
6+
7+
#include <opencv2/core.hpp>
8+
#include <spdlog/spdlog.h>
9+
10+
#include "xpano/utils/rect.h"
11+
#include "xpano/utils/vec.h"
12+
13+
namespace xpano::algorithm::crop {
14+
15+
namespace {
16+
struct Line {
17+
int start;
18+
int end;
19+
};
20+
21+
int Length(const Line& line) { return line.end - line.start; }
22+
23+
bool IsSet(unsigned char value) { return value == kMaskValueOn; }
24+
25+
std::optional<Line> FindLongestLineInColumn(cv::Mat column) {
26+
std::optional<Line> current_line;
27+
std::optional<Line> longest_line;
28+
29+
if (IsSet(column.at<unsigned char>(0, 0))) {
30+
current_line = Line{0, 0};
31+
}
32+
33+
for (int i = 1; i < column.rows; i++) {
34+
auto prev = IsSet(column.at<unsigned char>(i - 1, 0));
35+
auto current = IsSet(column.at<unsigned char>(i, 0));
36+
37+
if (!prev && current) {
38+
current_line = Line{i, i};
39+
}
40+
41+
if (prev && current) {
42+
current_line->end = i;
43+
}
44+
45+
if (prev && !current) {
46+
if (!longest_line || Length(*current_line) > Length(*longest_line)) {
47+
longest_line = current_line;
48+
}
49+
current_line.reset();
50+
}
51+
}
52+
53+
if (IsSet(column.at<unsigned char>(column.rows - 1, 0))) {
54+
if (!longest_line || Length(*current_line) > Length(*longest_line)) {
55+
longest_line = current_line;
56+
}
57+
}
58+
59+
return longest_line;
60+
}
61+
62+
} // namespace
63+
64+
// Approximage solution only.
65+
// Full solution would be https://stackoverflow.com/questions/2478447
66+
// This algorithm starts in the middle and expands the rectangle simultaneously
67+
// to the left and right. For the sake of simplicity if an empty column is
68+
// encountered, returns the current largest rectangle.
69+
std::optional<utils::RectPPi> FindLargestCrop(const cv::Mat& mask) {
70+
Line invalid_line = {mask.rows, 0};
71+
std::vector<Line> lines(mask.cols);
72+
for (int i = 0; i < mask.cols; i++) {
73+
auto longest_line = FindLongestLineInColumn(mask.col(i));
74+
lines[i] = longest_line.value_or(invalid_line);
75+
}
76+
77+
int half_size = mask.cols / 2;
78+
auto is_line_valid = [](const Line& line) { return line.start <= line.end; };
79+
if (mask.cols == 0 || !is_line_valid(lines[half_size])) {
80+
return {};
81+
}
82+
auto current_rect = utils::RectPPi{{half_size, lines[half_size].start},
83+
{half_size, lines[half_size].end}};
84+
auto largest_rect = current_rect;
85+
86+
int left_start = half_size - 1;
87+
int right_start = half_size + mask.cols % 2;
88+
for (int i = 0; i < half_size; i++) {
89+
auto left = left_start - i;
90+
auto right = right_start + i;
91+
auto left_line = lines[left];
92+
auto right_line = lines[right];
93+
94+
if (!is_line_valid(left_line)) {
95+
spdlog::warn("Auto crop: empty panorama at x = {}", left);
96+
return largest_rect;
97+
}
98+
99+
if (!is_line_valid(right_line)) {
100+
spdlog::warn("Auto crop: empty panorama at x = {}", right);
101+
return largest_rect;
102+
}
103+
104+
current_rect.start[0] = left;
105+
current_rect.end[0] = right;
106+
107+
if (int top = std::max(left_line.start, right_line.start);
108+
top > current_rect.start[1]) {
109+
current_rect.start[1] = top;
110+
}
111+
112+
if (int bottom = std::min(left_line.end, right_line.end);
113+
bottom < current_rect.end[1]) {
114+
current_rect.end[1] = bottom;
115+
}
116+
117+
if (utils::Area(current_rect) >= utils::Area(largest_rect)) {
118+
largest_rect = current_rect;
119+
}
120+
}
121+
122+
return largest_rect;
123+
}
124+
125+
} // namespace xpano::algorithm::crop

‎xpano/algorithm/auto_crop.h‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#pragma once
2+
3+
#include <optional>
4+
5+
#include <opencv2/core.hpp>
6+
7+
#include "xpano/utils/rect.h"
8+
9+
namespace xpano::algorithm::crop {
10+
11+
constexpr unsigned char kMaskValueOn = 0xFF;
12+
13+
std::optional<utils::RectPPi> FindLargestCrop(const cv::Mat& mask);
14+
15+
} // namespace xpano::algorithm::crop

0 commit comments

Comments
 (0)