Fix processing 8-bit RGB images

src/motiondetect_opt.c

@@ -24,6 +24,9 @@
  *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
  *
  */
+
+#include <limits.h>
+
 #include "motiondetect_opt.h"
 
 #ifdef USE_ORC
@@ -37,6 +40,21 @@
 #define SSE2_CMP_SUM_ROWS 8
 #endif
 
+unsigned int compareSubImg(unsigned char* const I1, unsigned char* const I2,
+                                    const Field* field, int width1, int width2, int height,
+                                    int bytesPerPixel, int d_x, int d_y,
+                                    unsigned int threshold){
+
+  int s2 = field->size / 2;
+
+  // check for frame boundaries. if field is outside frame after translation, return INT_MAX
+  if ((field->x - s2 + d_x + field->size > width2) || (field->y - s2 + d_y + field->size > height)){
+    return INT_MAX;
+  }
+
+  return _compareSubImg(I1, I2, field, width1, width2, height, bytesPerPixel, d_x, d_y, threshold);
+}
+
 #ifdef USE_SSE2
 /**
    \see contrastSubImg using SSE2 optimization, Planar (1 byte per channel) only

src/motiondetect_opt.h

@@ -31,15 +31,20 @@
 #include "motiondetect.h"
 
 #ifdef USE_SSE2_ASM //enable SSE2 inline asm code
-#define compareSubImg compareSubImg_thr_sse2_asm
+#define _compareSubImg compareSubImg_thr_sse2_asm
 #elif defined(USE_SSE2)      //enable SSE2 code
-#define compareSubImg compareSubImg_thr_sse2
+#define _compareSubImg compareSubImg_thr_sse2
 #elif defined(USE_ORC)
-#define compareSubImg compareSubImg_thr_orc
+#define _compareSubImg compareSubImg_thr_orc
 #else
-#define compareSubImg compareSubImg_thr
+#define _compareSubImg compareSubImg_thr
 #endif
 
+unsigned int compareSubImg(unsigned char* const I1, unsigned char* const I2,
+                                    const Field* field, int width1, int width2, int height,
+                                    int bytesPerPixel, int d_x, int d_y,
+                                    unsigned int threshold);
+
 #ifdef USE_SSE2
 double contrastSubImg1_SSE(unsigned char* const I, const Field* field,
                            int width, int height);

src/transformfixedpoint.c

@@ -238,11 +238,12 @@ inline void interpolateN(uint8_t *rv, fp16 x, fp16 y,
 {
   int32_t ix_f = fp16ToI(x);
   int32_t iy_f = fp16ToI(y);
-  if (ix_f < 0 || ix_f > width-1 || iy_f < 0 || iy_f > height - 1) {
+  int32_t ix_c = ix_f + 1;
+  int32_t iy_c = iy_f + 1;
+  if (ix_f < 0 || ix_f > width-1 || iy_f < 0 || iy_f > height - 1
+          || ix_c < 0 || ix_c > width-1 || iy_c < 0 || iy_c > height - 1) {
     *rv = def;
   } else {
-    int32_t ix_c = ix_f + 1;
-    int32_t iy_c = iy_f + 1;
     short v1 = PIXN(img, img_linesize, ix_c, iy_c, N, channel);
     short v2 = PIXN(img, img_linesize, ix_c, iy_f, N, channel);
     short v3 = PIXN(img, img_linesize, ix_f, iy_c, N, channel);
@@ -303,7 +304,7 @@ int transformPacked(VSTransformData* td, VSTransform t)
       fp16 y_s  = -zsin_a * x_d1 + zcos_a * y_d1 + c_ty;
 
       for (k = 0; k < channels; k++) { // iterate over colors
-        uint8_t *dest = &D_2[x + y * td->destbuf.linesize[0]+k];
+        uint8_t *dest = &D_2[x * channels + y * td->destbuf.linesize[0] + k];
         interpolateN(dest, x_s, y_s, D_1, td->src.linesize[0],
                      td->fiSrc.width, td->fiSrc.height,
                      channels, k, td->conf.crop ? 16 : *dest);

src/vidstabdefines.h

@@ -53,7 +53,7 @@
 #define PIXELN(img, linesize, x, y, w, h, N, channel, def) \
   (((x) < 0 || (y) < 0 || (x) >= (w) || (y) >= (h)) ? (def) : img[((x) + (y) * (linesize))*(N) + (channel)])
 /// pixel in N-channel image without rangecheck. channel in {0..N-1}
-#define PIXN(img, linesize, x, y, N, channel) (img[((x) + (y) * (linesize))*(N) + (channel)])
+#define PIXN(img, linesize, x, y, N, channel) (img[((x) * (N) + (y) * (linesize)) + (channel)])
 
 /**** Configurable memory and logging functions. Defined in libvidstab.c ****/