Extract disparity maps from the Pre-rendered Dataset

[bartn8]

Dear Authors,
Thanks for your very interesting work.

I'm trying to extract a disparity map from the Pre-rendered Dataset, however, I have some difficulties.

Firstly, given frame0, I tried to extract a depth map from data_0 -> denoising_position_map -> layer 0, however, the XYZ coordinates seem normalized between 0 and 1: to "resolve" the problem I applied the extrinsic_mat inside data_0.

Next, I used the c_x and stereo_interocular_distance [m] inside camera_info.json to convert the depth map to a disparity map.

Here there is an example using frame 2_118.

The colormap seems consistent, however, this procedure is wrong somewhere since the warped image is different w.r.t image_1

Do you have any suggestions?

Thanks for your time and support.

Best regards,
Luca

[behretj]

Were you able to resolve this anyhow? Be aware that disparity calculation requires horizontal focal length (fx) and not the x component of the principal point (cx) - maybe also just a typo on your side. However, I still wasn't able to obtain correct disparity/depth. For me it seems like that during the rendering process, depth was normalized, while camera pose and also stereo_interocular_distance were kept in metric scale, leading to the (arbitrary) offset you can recognize in your attached sample.
Any comment by the authors would be appreciated. For now, it seems like only re-rendering all scenes is the only option if depth is required?

[bartn8]

At the end I had to modify the rendering script, then re-rendering the scenes. Even so, I had a problem with a non-lambertian object rendered fully transparent. This object cannot be seen in the stereo pair, while having a very high disparity values in the labelled disparity map.

[behretj]

Thank you so much for your quick response! But did this fully transparent non-lambertian object happen a lot?
Or are most disparity maps now of good quality? If yes, is there any possibility to share the retrieved dataset somehow?
Thank you so much in advance!

[bartn8]

No, the object is like a long stick. But anyway, it could be better to figure out how to correct this, cause I think this will decrease the performance of the stereo model.

I attached here the git patch of my edits, but I don't remember the details.

diff --git a/syn_datagen/render.py b/syn_datagen/render.py
index 75aa700..3a64168 100644
--- a/syn_datagen/render.py
+++ b/syn_datagen/render.py
@@ -183,38 +183,38 @@ def dump_files(args, folder_name):
 
     shutil.move(os.path.join(args.output_folder, folder_name), os.path.join(args.output_folder, new_folder_name))
 
-    image1 = os.path.join(args.output_folder, new_folder_name, "frame0", "image_0.png")
-    image2 = os.path.join(args.output_folder, new_folder_name, "frame1", "image_0.png")
-    flow1 = os.path.join(args.output_folder, new_folder_name, "frame0", "denoising_vector_forward_0_0.png")
-    flow2 = os.path.join(args.output_folder, new_folder_name, "frame1", "denoising_vector_backward_0_0.png")
-    image1 = cv2.imread(image1)
-    image2 = cv2.imread(image2)
-    flow1 = cv2.imread(flow1)
-    flow2 = cv2.imread(flow2)
-
-    stereo1 = image1
-    stereo2 = os.path.join(args.output_folder, new_folder_name, "frame0", "image_1.png")
-    stereo2 = cv2.imread(stereo2)
-
-    image_concat = np.concatenate([image1, image2], axis=0)
-    cv2.imwrite(os.path.join(args.output_folder, new_folder_name, "image_concat.png"), image_concat)
-
-    flow_concat = np.concatenate([flow1, flow2], axis=0)
-    cv2.imwrite(os.path.join(args.output_folder, new_folder_name, "flow_concat.png"), flow_concat)
-
-    stereo_concat = np.concatenate([stereo1, stereo2], axis=1)
-    for i in range(0, stereo1.shape[0], 100):
-        cv2.line(stereo_concat, (0, i), (stereo_concat.shape[1], i), (0, 0, 0), 1)
-    cv2.imwrite(os.path.join(args.output_folder, new_folder_name, "stereo_concat.png"), stereo_concat)
-
-    # path = os.path.join(args.output_folder, new_folder_name, "frame0")
-    # for file in os.listdir(path):
-    #     if file.startswith("denoising") or file.startswith("gt"):
-    #         os.remove(os.path.join(path, file))
-    # path = os.path.join(args.output_folder, new_folder_name, "frame1")
-    # for file in os.listdir(path):
-    #     if file.startswith("denoising") or file.startswith("gt"):
-    #         os.remove(os.path.join(path, file))
+    # image1 = os.path.join(args.output_folder, new_folder_name, "frame0", "image_0.png")
+    # # image2 = os.path.join(args.output_folder, new_folder_name, "frame1", "image_0.png")
+    # # flow1 = os.path.join(args.output_folder, new_folder_name, "frame0", "denoising_vector_forward_0_0.png")
+    # # flow2 = os.path.join(args.output_folder, new_folder_name, "frame1", "denoising_vector_backward_0_0.png")
+    # image1 = cv2.imread(image1)
+    # # image2 = cv2.imread(image2)
+    # # flow1 = cv2.imread(flow1)
+    # # flow2 = cv2.imread(flow2)
+
+    # stereo1 = image1
+    # stereo2 = os.path.join(args.output_folder, new_folder_name, "frame0", "image_1.png")
+    # stereo2 = cv2.imread(stereo2)
+
+    # # image_concat = np.concatenate([image1, image2], axis=0)
+    # # cv2.imwrite(os.path.join(args.output_folder, new_folder_name, "image_concat.png"), image_concat)
+
+    # # flow_concat = np.concatenate([flow1, flow2], axis=0)
+    # # cv2.imwrite(os.path.join(args.output_folder, new_folder_name, "flow_concat.png"), flow_concat)
+
+    # stereo_concat = np.concatenate([stereo1, stereo2], axis=1)
+    # for i in range(0, stereo1.shape[0], 100):
+    #     cv2.line(stereo_concat, (0, i), (stereo_concat.shape[1], i), (0, 0, 0), 1)
+    # cv2.imwrite(os.path.join(args.output_folder, new_folder_name, "stereo_concat.jpg"), stereo_concat, [int(cv2.IMWRITE_JPEG_QUALITY), 50])
+
+    path = os.path.join(args.output_folder, new_folder_name, "frame0")
+    for file in os.listdir(path):
+        if file.startswith("denoising") or file.startswith("gt"):
+            os.remove(os.path.join(path, file))
+    # # path = os.path.join(args.output_folder, new_folder_name, "frame1")
+    # # for file in os.listdir(path):
+    # #     if file.startswith("denoising") or file.startswith("gt"):
+    # #         os.remove(os.path.join(path, file))
 
 def setup_gpu(args):
     bpy.context.preferences.addons["cycles"].preferences.compute_device_type = "CUDA"
@@ -261,7 +261,7 @@ def save_json_info(args, config, coef, folder_name):
     }
     json.dump(info, open(os.path.join(args.output_folder, folder_name, f"info.json"), 'w'), indent=4)
 
-gts = ["DENOISE"]
+gts = ["DEPTH"]
 def main(args):
     print(f'Generating {args.num_images} images for scene {args.scene_id}')
     config = json.load(open(f'scene_configs/{args.scene_id}.json', 'r'))
@@ -293,7 +293,7 @@ def main(args):
     for camera_index, (coef, next_coef) in enumerate(zip(camera_coefs, next_camera_coefs)):
         folder_name = folder_names[camera_index]
         os.makedirs(os.path.join(args.output_folder, folder_name, "frame0"), exist_ok=True)
-        os.makedirs(os.path.join(args.output_folder, folder_name, "frame1"), exist_ok=True)
+        #os.makedirs(os.path.join(args.output_folder, folder_name, "frame1"), exist_ok=True)
         scene.camera = cameras[camera_index]
         
         objects = None
@@ -302,7 +302,7 @@ def main(args):
         flying_objects.append(objects)
 
         render_frame(folder_name, scene, 0, "image")
-        render_frame(folder_name, scene, 1, "image")
+        #render_frame(folder_name, scene, 1, "image")
 
         adjust_object_visibility(objects, False)
         if not args.generate_gt:
@@ -329,7 +329,7 @@ def main(args):
 
         gt_init(scene, coef, gts=gts)        
         render_frame(folder_name, scene, 0, "gt", render_gt=True)
-        render_frame(folder_name, scene, 1, "gt", render_gt=True)
+        #render_frame(folder_name, scene, 1, "gt", render_gt=True)
         gt_finish(scene)
         
         adjust_object_visibility(objects, False)
diff --git a/syn_datagen/utils/blender_gt.py b/syn_datagen/utils/blender_gt.py
index 48a0328..a593f11 100644
--- a/syn_datagen/utils/blender_gt.py
+++ b/syn_datagen/utils/blender_gt.py
@@ -240,10 +240,10 @@ def save_data_to_npz(scene, is_stereo_activated, results, obj_poses):
         image_path = os.path.dirname(scene.render.filepath)
         save_img(result, image_path, id)
 
-        suffix = scene.render.views[id].file_suffix
-        data_path = os.path.join(image_path, f'data{suffix}.npz')
-        result_filtered = {k: v for k, v in result.items() if v is not None}
-        np.savez_compressed(data_path, **result_filtered)
+        # suffix = scene.render.views[id].file_suffix
+        # data_path = os.path.join(image_path, f'data{suffix}.npz')
+        # result_filtered = {k: v for k, v in result.items() if v is not None}
+        #np.savez_compressed(data_path, **result_filtered)
 
     cam0_result = results['cam0']
     save_results(cam0_result, 0)
diff --git a/syn_datagen/utils/utils.py b/syn_datagen/utils/utils.py
index fd1b55f..4731a0f 100644
--- a/syn_datagen/utils/utils.py
+++ b/syn_datagen/utils/utils.py
@@ -86,16 +86,28 @@ def mask_to_color(mask, color_seed=None):
 
     return segmentation_map_color
 
+def mycustom_depth_save(depth, path):
+    valid = (depth > 1e-3) & (depth < 65535 / 1000.0)
+    depth[~valid] = 0
+    cv2.imwrite(path, (depth * 1000).astype(np.uint16))
+
+def mycustom_disparity_save(disp, path):
+    valid = (disp > 1e-3) & (disp < 65535 / 256.0)
+    disp[~valid] = 0
+    cv2.imwrite(path, (disp * 256.0).astype(np.uint16))
+
 def save_img(out_dict, gt_dir_path, stereo_camera_id):
     depth = out_dict['depth_map'] if 'depth_map' in out_dict else None
     if depth is not None:
-        depth = depth_to_jet(depth)
-        cv2.imwrite(os.path.join(gt_dir_path, f'depth_{stereo_camera_id}.png'), depth)
+        # depth = depth_to_jet(depth)
+        # cv2.imwrite(os.path.join(gt_dir_path, f'depth_{stereo_camera_id}.png'), depth)
+        mycustom_depth_save(depth, os.path.join(gt_dir_path, f'depth_{stereo_camera_id}.png'))
 
     disp = out_dict['disparity_map'] if 'disparity_map' in out_dict else None
     if disp is not None:
-        disp = depth_to_jet(disp)
-        cv2.imwrite(os.path.join(gt_dir_path, f'disparity_{stereo_camera_id}.png'), disp)
+        #disp = depth_to_jet(disp)
+        # cv2.imwrite(os.path.join(gt_dir_path, f'disparity_{stereo_camera_id}.png'), disp)
+        mycustom_disparity_save(disp, os.path.join(gt_dir_path, f'disparity_{stereo_camera_id}.png'))
     
     seg = out_dict['segmentation_masks'] if 'segmentation_masks' in out_dict else None
     if seg is not None:

[bartn8]

P.S. This is the problem I was referring to: