On Mesh Alignment for Evaluation

[GCChen97]

Hi everyone. I just checked my evaluation code and found that the alignments require scaling and cropping the meshes, while my released dataset only contains the unpreprocessed data. Sorry for the mistake. Below, I show the steps for alignments:

1. Manually align (only R,t, no scale) the GT meshes with the predicted meshes in Blender
2. Label AABB for cropping predicted meshes in Blender
3. Crop and transform
   3.1 Crop meshes with bboxes
   3.2 Transform them to the frame of the GT meshes

Below is the code from step 3.

def icp(source, target, T_init):
    treg = o3d.pipelines.registration
    estimation = treg.TransformationEstimationPointToPoint(with_scaling=True)
    criteria = treg.ICPConvergenceCriteria(relative_fitness=0.000001,
                                        relative_rmse=0.000001,
                                        max_iteration=100)

    max_correspondence_distance = 0.07

    registration_icp = treg.registration_icp(source, target, max_correspondence_distance,
                                T_init, estimation, criteria)

    T_src2tgt = registration_icp.transformation
    return T_src2tgt

def o3d_pointcloud_from_mesh(mesh):
    pointcloud = o3d.geometry.PointCloud()
    pointcloud.points = mesh.vertices
    pointcloud.normals = mesh.compute_vertex_normals().vertex_normals

    return pointcloud

list_methods = [
    'PISR',
]

list_name_meshes_gt = [
    'black_loong',
    'red_loong',
    'standing_rabbit',
    'lying_rabbit',
]

dir_mesh = '/media/gccrcv/Data/Datasets/GCC/GT_mesh'
dir_bbox = '/media/gccrcv/Data/Workspace/pisr_datasets/bbox_predict'

dict_path_predict_meshes = {
    'PISR': [
    '/path/loongblack2/it20000-mc512.obj',
    '/path/loongred2/it20000-mc512.obj',
    '/path/rabbitstand/it20000-mc512.obj',
    '/path/rabbitdown/it20000-mc512.obj',
    ],
]

dict_meshes_gt = {}
dict_meshes_predict = {}
dict_tf_predict2gt = {}

for name_method in list_methods:
    print('Processing', name_method)
    os.makedirs(f'output2_{name_method}', exist_ok=True)
    for i in range(len(list_name_meshes_gt)):
        name_mesh = list_name_meshes_gt[i]
        list_path_meshes_predict = dict_path_predict_meshes[name_method]
        print('Processing', name_mesh)
        
        # load bbox of predicted mesh
        path_mesh_bbox = opj(dir_bbox, f'{name_mesh}-{name_method}-bbox.ply')
        if name_method in ['PISR','PISR*', 'PISR-',  'PISR*-', 'INeuS', 'INeuS+']:
            path_mesh_bbox = opj(dir_bbox, f'{name_mesh}-PISR-bbox.ply')
        mesh_bbox = o3d.io.read_triangle_mesh(path_mesh_bbox)
        bbox_predict_ori = mesh_bbox.get_axis_aligned_bounding_box()

        # load predicted mesh
        path_mesh_predict = list_path_meshes_predict[i]
        mesh_predict_ori = o3d.io.read_triangle_mesh(path_mesh_predict)
        if (name_method == 'nero' and name_mesh in ['red_loong', 'black_loong']) or (name_method == 'refneus' and name_mesh not in ['lying_rabbit',]):
            mesh_predict_ori.scale(4/3, center=(0, 0, 0))
        # break
        # crop and scale predicted mesh
        scale_down = 1/5
        bbox_predict = deepcopy(bbox_predict_ori)
        bbox_predict.translate(-mesh_bbox.get_center())
        bbox_predict.scale(scale_down, center=(0, 0, 0))
        
        mesh_predict = mesh_predict_ori.crop(bbox_predict_ori)
        mesh_predict.translate(-mesh_bbox.get_center())
        mesh_predict.scale(scale_down, center=(0, 0, 0))

        # load gt mesh
        path_mesh_gt = opj(dir_mesh, f'{name_mesh}-oriented-remesh_sharp9.ply')
        mesh_gt_ori = o3d.io.read_triangle_mesh(path_mesh_gt)
        # vis for check alignment initialization
        # o3d.io.write_triangle_mesh(f'output_{name_method}/mesh_predict.ply', mesh_predict)
        # o3d.io.write_triangle_mesh(f'output_{name_method}/mesh_gt_ori.ply', mesh_gt_ori)
        # break
        # ICP
        pc_predict = o3d_pointcloud_from_mesh(mesh_predict)
        pc_gt = o3d_pointcloud_from_mesh(mesh_gt_ori)
        T_init = np.identity(4, dtype=np.float32) # source to target
        T_eval = icp(pc_predict, pc_gt, T_init)

        # # ICP again
        mesh_gt = deepcopy(mesh_gt_ori)
        mesh_gt.transform(np.linalg.inv(T_eval))
        mesh_gt_crop = mesh_gt.crop(bbox_predict)
        mesh_gt_crop.transform(T_eval)
        pc_gt_cropped = o3d_pointcloud_from_mesh(mesh_gt_crop)
        T_eval1 = icp(pc_predict, pc_gt_cropped, T_eval)

        # # ICP last time
        mesh_gt_ori.transform(np.linalg.inv(T_eval))
        mesh_gt_ori_crop = mesh_gt_ori.crop(bbox_predict)
        mesh_gt_ori_crop.transform(T_eval)
        pc_gt_ori_cropped = o3d_pointcloud_from_mesh(mesh_gt_ori_crop)
        T_eval2 = icp(pc_predict, pc_gt_ori_cropped, T_eval1)
        
        # save
        mesh_predict.transform(T_eval2)
        dict_meshes_predict[name_mesh] = mesh_predict
        dict_meshes_gt[name_mesh] = mesh_gt_ori_crop
        dict_tf_predict2gt[name_mesh] = T_eval2
        o3d.io.write_triangle_mesh(f'output_{name_method}/{name_mesh}-cropped.ply', mesh_predict)
        o3d.io.write_triangle_mesh(f'output_{name_method}/{name_mesh}-gt-cropped.ply', mesh_gt_ori_crop)
        # break

    np.savez(f'output_{name_method}/align_pisr2gt.npz', **dict_tf_predict2gt) # These transformation omit the scaling and cropping.

[Yu-fei-Han]

Thank you for the explanation. Additionally, it would be very helpful if you could also provide the transformed gt_mesh files. This would make the dataset much more convenient for everyone to use when comparing different methods, and I believe it’s a more straightforward approach.