Rotating a normal map vector

[ghost]

The idea was to automate the transformation of camera space normal maps from a simple photometric stereo rig to object space, by utilizing the rotation matrix from a pose to rotate a vector encoded in RGB. The object space normal maps could then be projected onto the mesh and converted to tangent space later on. ÙwÓ

Example script 0

from concurrent.futures import ProcessPoolExecutor
import cv2
from functools import partial
import numpy as np
from os import cpu_count
from scipy.spatial.transform import Rotation

def rotateVector(normalMap, bitDepth, R):
  normalMap = np.dot((normalMap / bitDepth) * 2 - 1, R.T)
  return (normalMap / np.linalg.norm(normalMap, axis=2, keepdims=1) + 1) * 0.5 * bitDepth

def main():
  normalMap = np.array_split(cv2.imread("normal_map_1024_16bit.png", cv2.IMREAD_UNCHANGED), int(cpu_count() * 0.5))
  bitDepth = 65535 if normalMap[0].dtype == 'uint16' else 255
  
  for angle in range(360)[::10]:
    R = Rotation.from_euler('y', angle, degrees=True).as_matrix()
    
    normalMap_ = []
    with ProcessPoolExecutor() as executor:
      for result in executor.map(partial(rotateVector, bitDepth=bitDepth, R=R), normalMap):
        normalMap_.append(result)
    
    cv2.imwrite(f'pose_{angle}.png', np.concatenate(normalMap_).astype(normalMap[0].dtype))

if __name__ == '__main__':
  main()

Example script 1

from argparse import ArgumentParser
from concurrent.futures import ProcessPoolExecutor
import cv2
from functools import partial
from glob import glob
from json import load
import numpy as np
import os

def rotateVector(normalMap, bitDepth, R):
  normalMap = np.dot(((normalMap / bitDepth) * 2 - 1) * [-1, -1, 1], R)
  return (normalMap / np.linalg.norm(normalMap, axis=2, keepdims=1) + 1) * 0.5 * bitDepth

def main():
  parser = ArgumentParser()
  parser.add_argument('--input', type=str, required=True, help='path to the image directory')
  parser.add_argument('--sfm_data', type=str, required=True, help='path to the sfm_data JSON')
  args = parser.parse_args()
  
  os.chdir(args.input)
  
  outputDir = os.path.join(args.input, 'output')
  os.makedirs(outputDir, exist_ok = True)
  
  with open(args.sfm_data) as file:
    sfmData = load(file)
  
  filename, id_pose = [], []
  for view in sfmData['views']:
    filename.append(os.path.splitext(view['value']['ptr_wrapper']['data']['filename'])[0])
    id_pose.append(view['value']['ptr_wrapper']['data']['id_pose'])
  
  validPose = []
  for pose in sfmData['extrinsics']:
    validPose.append(pose['key'])
  
  intrinsics = sfmData['intrinsics'][0]['value']['ptr_wrapper']['data']
  cameraMatrix = np.array([[intrinsics['focal_length'], 0, intrinsics['principal_point'][0]], [0, intrinsics['focal_length'], intrinsics['principal_point'][1]], [0, 0, 1]])
  distCoeffs = (intrinsics['disto_k3'][0], intrinsics['disto_k3'][1], 0, 0, intrinsics['disto_k3'][2])
  
  views = []
  for file in glob('*.png'):
    index = id_pose[filename.index(os.path.splitext(os.path.basename(file))[0])]
    views.append(validPose.index(index))
    R = np.rot90(sfmData['extrinsics'][validPose.index(index)]['value']['rotation'], 2)
    normalMap = np.array_split(cv2.imread(file, cv2.IMREAD_UNCHANGED), int(os.cpu_count() * 0.5))
    bitDepth = 65535 if normalMap[0].dtype == 'uint16' else 255
    
    normalMap_ = []
    with ProcessPoolExecutor() as executor:
      for result in executor.map(partial(rotateVector, bitDepth=bitDepth, R=R), normalMap):
        normalMap_.append(result)
    
    dst = cv2.undistort(np.concatenate(normalMap_).astype(normalMap[0].dtype), cameraMatrix, distCoeffs)
    cv2.imwrite(os.path.join(outputDir, os.path.splitext(os.path.basename(file))[0]+'.png'), dst)
  
  print('Saving views to views.txt')
  with open(os.path.join(outputDir, 'views.txt'), 'w') as file2:
    for view in views:
      file2.write(f'{view}\n')

if __name__ == '__main__':
  main()

[cdcseacave]

Can you pls reformulate what is exactly that you try to achieve?

[ghost]

Sure!

[ghost]

Photometric stereo script

[1] https://github.com/NamanMakkar/Stereo

from argparse import ArgumentParser
from concurrent.futures import ThreadPoolExecutor
import cv2
from glob import glob
import numpy as np
from os import chdir
import sys

def loadImage(image):
  return np.float32(cv2.imread(image, cv2.IMREAD_UNCHANGED))

def photometricStereo(lights, images):
  height, width, channel = images[0].shape
  albedo = np.zeros((height, width, channel), dtype=np.float32)
  normals = np.zeros((height, width, 3), dtype=np.float32)
  images_np = np.array(images)
  
  G_first_term = np.linalg.inv(lights.T @ lights)
  
  I = images_np.reshape(len(images), height, width, channel)
  
  G = np.tensordot(G_first_term, np.tensordot(lights, I, axes=([0], [0])), axes=([1], [0]))
  
  k_d = np.linalg.norm(G, axis=0)
  
  mask = k_d < 1e-7
  albedo[mask] = 0
  
  k_d[mask] = np.inf
  
  normals = G / k_d[np.newaxis, :, :, :]
  
  normals[:, k_d == np.inf] = 0
  
  normals = np.transpose(normals[:, :, :, 0], (1, 2, 0))
  
  albedo[~mask] = k_d[~mask]
  
  return albedo, normals

if __name__ == '__main__':
  parser = ArgumentParser()
  parser.add_argument('--input', type=str, required=True, help='path to the image directory')
  parser.add_argument('--extension', type=str, default='png', help='image extension')
  parser.add_argument('--lights', type=str, required=True, help='path to the lights file')
  parser.add_argument('--albedo_output', type=str, default='albedo.png', help='path to the output image file')
  parser.add_argument('--normals_output', type=str, default='normals.png', help='path to the output image file')

  args = parser.parse_args()
  
  chdir(args.input)
  
  lights = np.loadtxt(args.lights)
  
  imageList = glob(f'*.{args.extension}')
  listRemove = ['mask', 'albedo', 'normals']
  for fileName in listRemove:
    if f'{fileName}.{args.extension}' in imageList:
      imageList.remove(f'{fileName}.{args.extension}')
  
  if len(imageList) != lights.shape[0]:
    print('Error: The number of light vectors does not match the number of input images.')
    sys.exit()
  
  image = cv2.imread(imageList[0], cv2.IMREAD_UNCHANGED)
  bitDepth = 65535 if image.dtype == 'uint16' else 255
  
  with ThreadPoolExecutor() as executor:
    images = [result for result in executor.map(loadImage, imageList)]
  
  albedo, normals = photometricStereo(lights, images)
  
  albedo = np.clip(albedo, 0, bitDepth).astype(image.dtype)
  normals = cv2.cvtColor(((normals + 1) * 0.5 * bitDepth).astype(image.dtype), cv2.COLOR_RGB2BGR)

  print(f'Saving albedo to {args.albedo_output}')
  cv2.imwrite(args.albedo_output, albedo)
  print(f'Saving normals to {args.normals_output}')
  cv2.imwrite(args.normals_output, normals)

[ghost]

Light calibration script

from argparse import ArgumentParser
from concurrent.futures import ThreadPoolExecutor
import cv2
from glob import glob
import numpy as np
from os import chdir

def spherePos(image):
  a = np.where(image == image.max())
  x1, y1, x2, y2 = np.min(a[1]), np.min(a[0]), np.max(a[1]), np.max(a[0])
  radius = [(x2 - x1) * 0.5, (y2 - y1) * 0.5]
  center = [x2 - radius[0], y2 - radius[1]]
  return radius, center

def imageProcess(image):
  image = cv2.imread(image, 0)
  mask = cv2.imread('mask.png', 0)
  imageMasked = cv2.bitwise_and(image,image, mask=mask)
  
  sR, sC = spherePos(mask)
  lR, lC = spherePos(imageMasked)
  
  X, Y = np.subtract(lC, sC) * [1, -1] / sR
  Z = np.sqrt(1 - pow(X, 2) - pow(Y, 2))
  
  return [X, Y, Z]

if __name__ == '__main__':
  parser = ArgumentParser()
  parser.add_argument('--input', type=str, required=True, help='path to the image directory')
  parser.add_argument('--extension', type=str, default='png', help='image extension')
  parser.add_argument('--lights_output', type=str, default='lights.txt', help='path to the output text file')
  args = parser.parse_args()
  
  chdir(args.input)
  
  imageList = glob(f'*.{args.extension}')
  listRemove = ['mask', 'albedo', 'normals']
  for fileName in listRemove:
    if f'{fileName}.{args.extension}' in imageList:
      imageList.remove(f'{fileName}.{args.extension}')
  
  with ThreadPoolExecutor() as executor:
    L = [result for result in executor.map(imageProcess, imageList)]
  
  print(f'Saving lights to {args.lights_output}')
  with open(args.lights_output, 'w') as file:
    for X, Y, Z in L:
      file.write(f'{X} {Y} {Z}\n')