Ambient dataset cleanup and upload (#148)

* Accept relative paths for reference background.

* Check background files in dataset clean up.

* Decrease lower level.

* Add config.

* Fix file deletion.

* Change lower limit.

* Remove changes in train script.

* Remove changes in training.

* Remove unused import.

* Put back flips.

* Clean up batch parsing.

configs/analyze_dataset.yaml

@@ -4,7 +4,7 @@ hydra:
     chdir: True    # change to output folder
 
 dataset_path: null
-desired_range: [150, 255]
+desired_range: [10, 255]
 saturation_percent: 0.05
 delete_bad: False
 n_files: null

configs/upload_multilens_mirflickr_ambient.yaml

@@ -0,0 +1,20 @@
+# python scripts/data/upload_dataset_huggingface.py -cn upload_multilens_mirflickr_ambient
+defaults:
+  - upload_dataset_huggingface
+  - _self_
+
+repo_id: "Lensless/MultiLens-Mirflickr-Ambient"
+n_files:
+test_size: 0.15
+lensless:
+  dir: /dev/shm/all_measured_20240813-183259
+  ambient: True
+  ext: ".png"
+
+lensed:
+  dir: data/mirflickr/mirflickr
+  ext: ".jpg"
+
+files:
+  psf: data/multilens_psf.png
+  measurement_config: data/collect_dataset_background_multilens.yaml

lensless/recon/utils.py

@@ -878,24 +878,13 @@ def train_epoch(self, data_loader):
             X = batch[0].to(self.device)
             y = batch[1].to(self.device)
             if self.train_multimask or self.train_random_flip:
-                psfs = batch[2]
-                psfs = psfs.to(self.device)
+                psfs = batch[2].to(self.device)
             else:
                 psfs = None
             if self.train_random_flip:
                 flip_lr = batch[3]
                 flip_ud = batch[4]
 
-            # if self.train_random_flip:
-            #     X, y, psfs, flip_lr, flip_ud = batch
-            #     psfs = psfs.to(self.device)
-            # elif self.train_multimask:
-            #     X, y, psfs = batch
-            #     psfs = psfs.to(self.device)
-            # else:
-            #     X, y = batch
-            #     psfs = None
-
             random_rotate = False
             if self.random_rotate:
                 random_rotate = np.random.uniform(-self.random_rotate, self.random_rotate)

scripts/measure/analyze_measured_dataset.py

@@ -2,7 +2,7 @@
 Check maximum pixel value of images and check for saturation / underexposure.
 
 ```
-python scripts/measure/analyze_measured_dataset.py folder=PATH
+python scripts/measure/analyze_measured_dataset.py dataset_path=PATH
 ```
 """
 
@@ -123,6 +123,31 @@ def analyze_dataset(config):
         else:
             print("Not deleting bad files")
 
+    # check for matching background file
+    files_bg = natural_sort(glob.glob(os.path.join(folder, "black_background*.png")))
+    # -- remove files_bg from files
+    files = [fn for fn in files if fn not in files_bg]
+
+    if len(files_bg) > 0:
+        print("Found {} background files".format(len(files_bg)))
+        # detect files that don't have background
+        files_no_bg = []
+        for fn in files:
+            bn = os.path.basename(fn).split(".")[0]
+            _bg_file = os.path.join(folder, "black_background{}.png".format(bn))
+            if _bg_file not in files_bg:
+                files_no_bg.append(fn)
+
+        print("Found {} files without background".format(len(files_no_bg)))
+        # ask to delete files without background
+        response = None
+        while response not in ["yes", "no"]:
+            response = input("Delete files without background: [yes|no] : ")
+        if response == "yes":
+            for _fn in files_no_bg:
+                if os.path.exists(_fn):  # maybe already deleted before
+                    os.remove(_fn)
+
 
 if __name__ == "__main__":
     analyze_dataset()

scripts/recon/train_learning_based.py

@@ -31,7 +31,7 @@
 import wandb
 import logging
 import hydra
-from hydra.utils import get_original_cwd, to_absolute_path
+from hydra.utils import get_original_cwd
 import os
 import numpy as np
 import time
@@ -227,7 +227,7 @@ def train_learned(config):
                 display_res=config.files.image_res,
                 alignment=config.alignment,
                 bg_snr_range=config.files.background_snr_range,  # TODO check if correct
-                bg_fp=to_absolute_path(config.files.background_fp),
+                bg_fp=config.files.background_fp,
             )
 
         else:
@@ -251,7 +251,7 @@ def train_learned(config):
                 simulate_lensless=config.files.simulate_lensless,
                 random_flip=config.files.random_flip,
                 bg_snr_range=config.files.background_snr_range,
-                bg_fp=to_absolute_path(config.files.background_fp),
+                bg_fp=config.files.background_fp,
             )
 
         test_set = HFDataset(
@@ -271,7 +271,7 @@ def train_learned(config):
             n_files=config.files.n_files,
             simulation_config=config.simulation,
             bg_snr_range=config.files.background_snr_range,
-            bg_fp=to_absolute_path(config.files.background_fp),
+            bg_fp=config.files.background_fp,
             force_rgb=config.files.force_rgb,
             simulate_lensless=False,  # in general evaluate on measured (set to False)
         )