From 7695b43f8051f6a79f92b8a9a6b43ef56feadee3 Mon Sep 17 00:00:00 2001 From: arbitularov Date: Tue, 5 Feb 2019 21:56:55 +0300 Subject: [PATCH] Add bicubic interpolation for PyTorch --- training/bicubic.py | 251 +++++++++++++++++++++++++++++++++++++++++++ training/datasets.py | 2 +- training/losses.py | 3 +- training/models.py | 5 +- 4 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 training/bicubic.py diff --git a/training/bicubic.py b/training/bicubic.py new file mode 100644 index 0000000..835b6c0 --- /dev/null +++ b/training/bicubic.py @@ -0,0 +1,251 @@ +''' +import sys, time +import torch +import numpy as np +from torch.autograd import Variable + + +# Interpolation kernel +def u(s,a): + if (abs(s) >=0) & (abs(s) <=1): + return (a+2)*(abs(s)**3)-(a+3)*(abs(s)**2)+1 + elif (abs(s) > 1) & (abs(s) <= 2): + return a*(abs(s)**3)-(5*a)*(abs(s)**2)+(8*a)*abs(s)-4*a + return 0 + +#Paddnig +def padding(img,B,C,H,W): + zimg = np.zeros([B,C,H+4,W+4]) + zimg[:,:C,2:H+2,2:W+2] = img + #Pad the first/last two col and row + zimg[:,:C,2:H+2,0:2]=img[:,:C,:,0:1] + zimg[:,:,H+2:H+4,2:W+2]=img[:,:,H-1:H,:] + zimg[:,:,2:H+2,W+2:W+4]=img[:,:,:,W-1:W] + zimg[:,:C,0:2,2:W+2]=img[:,:C,0:1,:] + #Pad the missing eight points + zimg[0,:C,0:2,0:2]=img[0,:C,0,0] + zimg[0,:C,H+2:H+4,0:2]=img[0,:C,H-1,0] + zimg[0,:C,H+2:H+4,W+2:W+4]=img[0,:C,H-1,W-1] + zimg[0,:C,0:2,W+2:W+4]=img[0,:C,0,W-1] + return zimg + +# Bicubic operation +def bicubic(img): + + a = -0.75 + + img = img.cpu().detach().numpy() + #Get image size + B, C, H, W = img.shape + + + img = padding(img,B,C,H,W) + + + #Create new image + dH = 34 + dW = 34 + dst = np.zeros((B, C, dH, dW)) + + + h = 1/(34/H) + + for b in range(B): + for c in range(C): + for j in range(dH): + for i in range(dW): + x, y = i * h + 2 , j * h + 2 + + x1 = 1 + x - math.floor(x) + x2 = x - math.floor(x) + x3 = math.floor(x) + 1 - x + x4 = math.floor(x) + 2 - x + + y1 = 1 + y - math.floor(y) + y2 = y - math.floor(y) + y3 = math.floor(y) + 1 - y + y4 = math.floor(y) + 2 - y + + mat_l = np.matrix([[u(x1,a),u(x2,a),u(x3,a),u(x4,a)]]) + mat_m = np.matrix([[img[0,c,int(y-y1),int(x-x1)],img[0,c,int(y-y2),int(x-x1)],img[0,c,int(y+y3),int(x-x1)],img[0,c,int(y+y4),int(x-x1)]], + [img[0,c,int(y-y1),int(x-x2)],img[0,c,int(y-y2),int(x-x2)],img[0,c,int(y+y3),int(x-x2)],img[0,c,int(y+y4),int(x-x2)]], + [img[0,c,int(y-y1),int(x+x3)],img[0,c,int(y-y2),int(x+x3)],img[0,c,int(y+y3),int(x+x3)],img[0,c,int(y+y4),int(x+x3)]], + [img[0,c,int(y-y1),int(x+x4)],img[0,c,int(y-y2),int(x+x4)],img[0,c,int(y+y3),int(x+x4)],img[0,c,int(y+y4),int(x+x4)]]]) + mat_r = np.matrix([[u(y1,a)],[u(y2,a)],[u(y3,a)],[u(y4,a)]]) + dst[b, c, j, i] = np.asarray( np.dot( np.dot(mat_l, mat_m), mat_r) ) + + dst = torch.Tensor(dst).cuda() + dst = dst.type(torch.cuda.FloatTensor) + dst = Variable(dst, requires_grad=True).cuda() + return dst + + + +''' + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +import math +import torch +import numpy as np +from torch.autograd import Variable +from numba import jit + +# Interpolation kernel +@jit +def u(l1, l2, l3, l4,a): + x1 = abs(l1) + x2 = abs(l2) + x3 = abs(l3) + x4 = abs(l4) + + y1 = 0 + if (x1 >=0) & (x1 <=1): + + y1 = (a+2)*(x1**3)-(a+3)*(x1**2)+1 + elif (x1 > 1) & (x1 <= 2): + y1 = a*(x1**3)-(5*a)*(x1**2)+(8*a)*x1-4*a + + + y2 = 0 + if (x2 >=0) & (x2 <=1): + + y2 = (a+2)*(x2**3)-(a+3)*(x2**2)+1 + elif (x2 > 1) & (x2 <= 2): + y2 = a*(x2**3)-(5*a)*(x2**2)+(8*a)*x2-4*a + + + y3 = 0 + if (x3 >=0) & (x3 <=1): + + y3 = (a+2)*(x3**3)-(a+3)*(x3**2)+1 + elif (x3 > 1) & (x3 <= 2): + y3 = a*(x3**3)-(5*a)*(x3**2)+(8*a)*x3-4*a + + + y4 = 0 + if (x4 >=0) & (x4 <=1): + + y4 = (a+2)*(x4**3)-(a+3)*(x4**2)+1 + elif (x4 > 1) & (x4 <= 2): + y4 = a*(x4**3)-(5*a)*(x4**2)+(8*a)*x4-4*a + + + return y1, y2, y3, y4 + +@jit +def d(mat): + mat_l, mat_m, mat_r = mat + d = np.dot( np.dot(mat_l, mat_m), mat_r) + return d + +@jit +def mat(h,a,i,j,c,img): + x, y = i * h + 2 , j * h + 2 + + fx = math.floor(x) + x1 = 1 + x -fx + x2 = x - fx + x3 = fx + 1 - x + x4 = fx + 2 - x + + fy = math.floor(y) + y1 = 1 + y - fy + y2 = y - fy + y3 = fy + 1 - y + y4 = fy + 2 - y + + ny1 = int(y-y1) + ny2 = int(y-y2) + ny3 = int(y+y3) + ny4 = int(y+y4) + nmx1 = int(x-x1) + nmx2 = int(x-x2) + npx3 = int(x+x3) + npx4 = int(x+x4) + + mat_m = np.array([[img[0,c,ny1,nmx1],img[0,c,ny2,nmx1],img[0,c,ny3,nmx1],img[0,c,ny4,nmx1]], + [img[0,c,ny1,nmx2],img[0,c,ny2,nmx2],img[0,c,ny3,nmx2],img[0,c,ny4,nmx2]], + [img[0,c,ny1,npx3],img[0,c,ny2,npx3],img[0,c,ny3,npx3],img[0,c,ny4,npx3]], + [img[0,c,ny1,npx4],img[0,c,ny2,npx4],img[0,c,ny3,npx4],img[0,c,ny4,npx4]]]) + + p1, p2, p3, p4 = u(y1, y2, y3, y4,a) + + + return np.array([[u(x1, x2, x3, x4,a)]]),mat_m, np.array([[p1],[p2],[p3],[p4]]) + + +#Paddnig +def padding(img,B,C,H,W): + zimg = np.zeros([B,C,H+4,W+4]) + zimg[:,:C,2:H+2,2:W+2] = img + #Pad the first/last two col and row + zimg[:,:C,2:H+2,0:2]=img[:,:C,:,0:1] + zimg[:,:,H+2:H+4,2:W+2]=img[:,:,H-1:H,:] + zimg[:,:,2:H+2,W+2:W+4]=img[:,:,:,W-1:W] + zimg[:,:C,0:2,2:W+2]=img[:,:C,0:1,:] + #Pad the missing eight points + zimg[0,:C,0:2,0:2]=img[0,:C,0,0] + zimg[0,:C,H+2:H+4,0:2]=img[0,:C,H-1,0] + zimg[0,:C,H+2:H+4,W+2:W+4]=img[0,:C,H-1,W-1] + zimg[0,:C,0:2,W+2:W+4]=img[0,:C,0,W-1] + return zimg + +# Bicubic operation +def bicubic(img, newSize, a = -0.75): + + # Coefficient + img = img.cpu().detach().numpy() + + + #Get image size + B, C, H, W = img.shape + + img = padding(img,B,C,H,W) + + #Create new image + dH = newSize + dW = newSize + dst = np.zeros([B, C, dH, dW]) + + h = 1/(newSize/H) + + for b in range(B): + for c in range(C): + for j in range(dH): + for i in range(dW): + + dst[b, c, j, i] = d(mat(h,a,i,j,c,img)) + + dst = torch.Tensor(dst).cuda() + dst = dst.type(torch.cuda.FloatTensor) + dst = Variable(dst, requires_grad=True).cuda() + return dst diff --git a/training/datasets.py b/training/datasets.py index fa4f07f..fafb9f7 100644 --- a/training/datasets.py +++ b/training/datasets.py @@ -36,7 +36,7 @@ class ImageNetVID(Dataset): """ def __init__(self, imagenet_dir, transforms=ToTensor(), - reference_size=127, search_size=255, final_size=33, + reference_size=127, search_size=255, final_size=129, label_fcn=BCELoss, upscale_factor=4, max_frame_sep=50, pos_thr=25, neg_thr=50, cxt_margin=0.5, single_label=True, img_read_fcn=imread, diff --git a/training/losses.py b/training/losses.py index 1965f33..3ec4296 100644 --- a/training/losses.py +++ b/training/losses.py @@ -14,7 +14,8 @@ def BCELogit_Loss(score_map, labels): loss (scalar torch.Tensor): The BCE Loss with Logits for the score map and labels. """ labels = labels.unsqueeze(1) + #UserWarning: reduction='elementwise_mean' is deprecated, please use reduction='mean' instead. warnings.warn("reduction='elementwise_mean' is deprecated, please use reduction='mean' instead.") loss = F.binary_cross_entropy_with_logits(score_map, labels[:, :, :, :, 0], weight=labels[:, :, :, :, 1], - reduction='elementwise_mean') + reduction='mean') return loss diff --git a/training/models.py b/training/models.py index d62e459..162542c 100644 --- a/training/models.py +++ b/training/models.py @@ -10,6 +10,7 @@ import torch.nn.functional as F from torch.nn.init import xavier_uniform_, constant_, zeros_, normal_ +from training.bicubic import bicubic class BaselineEmbeddingNet(nn.Module): """ Definition of the embedding network used in the baseline experiment of @@ -231,8 +232,8 @@ def match_corr(self, embed_ref, embed_srch): match_map = match_map.permute(1, 0, 2, 3) match_map = self.match_batchnorm(match_map) if self.upscale: - match_map = F.interpolate(match_map, self.upsc_size, mode='bilinear', - align_corners=False) + #match_map = F.interpolate(match_map, self.upsc_size, mode='bilinear', align_corners=False) + match_map = bicubic(match_map, self.upsc_size) return match_map