Merge pull request #24 from osamu620/develop

Reduce the number of Interleaved Store & Load in color conversion and subsampling

CMakeLists.txt

@@ -38,7 +38,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU")
 	set(CMAKE_CXX_FLAGS "-Wall -Wextra -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer")
 	set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -O0 -g -fsanitize=address")
 	set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} -O3 -DNDEBUG")
-	set(CMAKE_CXX_FLAGS_RelWithDebInfo "${CMAKE_CXX_FLAGS} -O2 -g -DNDEBUG")
+	set(CMAKE_CXX_FLAGS_RelWithDebInfo "${CMAKE_CXX_FLAGS} -O3 -g -DNDEBUG")
 endif()
 
 if(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") # MSVC

lib/block_coding_256.cpp

@@ -72,7 +72,7 @@ HWY_ALIGN uint64_t shift[4] = {24, 16, 8, 0};
 const auto vs               = Load(u64, shift);
 a0                          = Shl(a0, vs);
 a1                          = Shl(a1, vs);
-bitmap                      = (GetLane(SumOfLanes(u64, a0)) << 32) + GetLane(SumOfLanes(u64, a1));
+bitmap                      = (ReduceSum(u64, a0) << 32) | ReduceSum(u64, a1);
 
 auto abs_row01 = Abs(row01);
 auto abs_row23 = Abs(row23);

lib/block_coding_512.cpp

@@ -42,7 +42,7 @@ auto a0                   = SumsOf8(bitmap_rows_76543210);
 HWY_ALIGN uint64_t shift[8] = {56, 48, 40, 32, 24, 16, 8, 0};
 const auto vs               = Load(u64, shift);
 a0                          = Shl(a0, vs);
-bitmap                      = GetLane(SumOfLanes(u64, a0));
+bitmap                      = ReduceSum(u64, a0);
 
 auto abs_row0123 = Abs(row0123);
 auto abs_row4567 = Abs(row4567);

lib/color.hpp

@@ -3,6 +3,6 @@
 #include <vector>
 
 namespace jpegenc_hwy {
-void rgb2ycbcr(uint8_t *in, int width);
-void subsample(uint8_t *in, std::vector<int16_t *> &out, int width, int YCCtype);
+void rgb2ycbcr(uint8_t *in, std::vector<uint8_t *> &out, int width);
+void subsample(std::vector<uint8_t *>, std::vector<int16_t *> &out, int width, int YCCtype);
 }  // namespace jpegenc_hwy

lib/jpegenc.cpp

@@ -25,9 +25,11 @@ class jpeg_encoder_impl {
   int YCCtype;
   const int rounded_width;
   const int rounded_height;
-  std::vector<std::unique_ptr<int16_t[], hwy::AlignedFreer>> line_buffer;
+  std::vector<std::unique_ptr<uint8_t[], hwy::AlignedFreer>> line_buffer0;
+  std::vector<std::unique_ptr<int16_t[], hwy::AlignedFreer>> line_buffer1;
   std::unique_ptr<int16_t[], hwy::AlignedFreer> mcu_buffer;
-  std::vector<int16_t *> yuv;
+  std::vector<uint8_t *> yuv0;
+  std::vector<int16_t *> yuv1;
   int16_t *mcu;
   HWY_ALIGN int16_t qtable[DCTSIZE2 * 2];
   bitstream enc;
@@ -43,33 +45,44 @@ class jpeg_encoder_impl {
         YCCtype(ycc),
         rounded_width(round_up(inimg.width, DCTSIZE * (YCC_HV[YCCtype][0] >> 4))),
         rounded_height(round_up(inimg.height, DCTSIZE * (YCC_HV[YCCtype][0] & 0xF))),
-        line_buffer(ncomp),
-        yuv(ncomp),
+        line_buffer0(ncomp),
+        line_buffer1(ncomp),
+        yuv0(ncomp),
+        yuv1(ncomp),
         qtable{0},
         enc(3000000),
         use_RESET(false) {
-    int nc = inimg.nc;
-    if (nc == 1) {
+    int ncomp_out = inimg.nc;
+    if (ncomp_out == 1) {
       YCCtype = YCC::GRAY;
     }
-    nc                     = (YCCtype == YCC::GRAY2) ? 1 : nc;
+    ncomp_out              = (YCCtype == YCC::GRAY2) ? 1 : ncomp_out;
     const int scale_x      = YCC_HV[YCCtype][0] >> 4;
     const int scale_y      = YCC_HV[YCCtype][0] & 0xF;
     const size_t bufsize_L = rounded_width * LINES;
     const size_t bufsize_C = rounded_width / scale_x * LINES / scale_y;
 
     // Prepare line-buffers
-    line_buffer[0] = hwy::AllocateAligned<int16_t>(bufsize_L);
-    for (size_t c = 1; c < line_buffer.size(); ++c) {
-      line_buffer[c] = hwy::AllocateAligned<int16_t>(bufsize_C);
+    line_buffer0[0] = hwy::AllocateAligned<uint8_t>(bufsize_L);
+    for (int c = 1; c < ncomp; ++c) {
+      line_buffer0[c] = hwy::AllocateAligned<uint8_t>(bufsize_L);
     }
-    yuv[0] = line_buffer[0].get();
-    for (int c = 1; c < nc; ++c) {
-      yuv[c] = line_buffer[c].get();
+    yuv0[0] = line_buffer0[0].get();
+    for (int c = 1; c < ncomp; ++c) {
+      yuv0[c] = line_buffer0[c].get();
+    }
+
+    line_buffer1[0] = hwy::AllocateAligned<int16_t>(bufsize_L);
+    for (size_t c = 1; c < line_buffer1.size(); ++c) {
+      line_buffer1[c] = hwy::AllocateAligned<int16_t>(bufsize_C);
+    }
+    yuv1[0] = line_buffer1[0].get();
+    for (int c = 1; c < ncomp_out; ++c) {
+      yuv1[c] = line_buffer1[c].get();
     }
 
     // Prepare mcu-buffers
-    const int c = (nc == 1) ? 1 : 0;
+    const int c = (ncomp_out == 1) ? 1 : 0;
     mcu_buffer  = hwy::AllocateAligned<int16_t>(DCTSIZE2 * scale_x * scale_y + ((DCTSIZE2 * 2) >> c));
     mcu         = mcu_buffer.get();
   }
@@ -93,10 +106,12 @@ class jpeg_encoder_impl {
     // Loop of 16 pixels height
     for (int n = 0; n < rounded_height - LINES; n += LINES) {
       if (ncomp == 3) {
-        jpegenc_hwy::rgb2ycbcr(src, rounded_width);
+        jpegenc_hwy::rgb2ycbcr(src, yuv0, rounded_width);
+      } else {
+        yuv0[0] = src;
       }
-      jpegenc_hwy::subsample(src, yuv, rounded_width, YCCtype);
-      jpegenc_hwy::encode_lines(yuv, mcu, rounded_width, LINES, YCCtype, qtable, prev_dc, tab_Y, tab_C,
+      jpegenc_hwy::subsample(yuv0, yuv1, rounded_width, YCCtype);
+      jpegenc_hwy::encode_lines(yuv1, mcu, rounded_width, LINES, YCCtype, qtable, prev_dc, tab_Y, tab_C,
                                 enc);
       // RST marker insertion, if any
       if (use_RESET) {
@@ -109,10 +124,12 @@ class jpeg_encoder_impl {
     const int last_mcu_height = (rounded_height % LINES) ? DCTSIZE : LINES;
 
     if (ncomp == 3) {
-      jpegenc_hwy::rgb2ycbcr(src, rounded_width);
+      jpegenc_hwy::rgb2ycbcr(src, yuv0, rounded_width);
+    } else {
+      yuv0[0] = src;
     }
-    jpegenc_hwy::subsample(src, yuv, rounded_width, YCCtype);
-    jpegenc_hwy::encode_lines(yuv, mcu, rounded_width, last_mcu_height, YCCtype, qtable, prev_dc, tab_Y,
+    jpegenc_hwy::subsample(yuv0, yuv1, rounded_width, YCCtype);
+    jpegenc_hwy::encode_lines(yuv1, mcu, rounded_width, last_mcu_height, YCCtype, qtable, prev_dc, tab_Y,
                               tab_C, enc);
 
     // Finalize codestream