Change dataset check on saturation ratio.

configs/analyze_dataset.yaml

@@ -4,7 +4,8 @@ hydra:
     chdir: True    # change to output folder
 
 dataset_path: null
-desired_range: [150, 254]
+desired_range: [150, 255]
+saturation_percent: 0.05
 delete_bad: False
 n_files: null
 start_idx: null

scripts/measure/analyze_measured_dataset.py

@@ -15,6 +15,19 @@
 import matplotlib.pyplot as plt
 import time
 import tqdm
+import re
+
+
+def convert(text):
+    return int(text) if text.isdigit() else text.lower()
+
+
+def alphanum_key(key):
+    return [convert(c) for c in re.split("([0-9]+)", key)]
+
+
+def natural_sort(arr):
+    return sorted(arr, key=alphanum_key)
 
 
 @hydra.main(version_base=None, config_path="../../configs", config_name="analyze_dataset")
@@ -24,13 +37,14 @@ def analyze_dataset(config):
     desired_range = config.desired_range
     delete_bad = config.delete_bad
     start_idx = config.start_idx
+    saturation_percent = config.saturation_percent
 
     assert (
         folder is not None
     ), "Must specify folder to analyze in config or through command line (folder=PATH)."
 
     # get all PNG files in folder
-    files = sorted(glob.glob(os.path.join(folder, "*.png")))
+    files = natural_sort(glob.glob(os.path.join(folder, "*.png")))
     print("Found {} files".format(len(files)))
     if start_idx is not None:
         files = files[start_idx:]
@@ -48,10 +62,9 @@ def analyze_dataset(config):
         im = np.array(Image.open(fn))
         max_val = im.max()
         max_vals.append(max_val)
+        saturation_ratio = np.sum(im >= desired_range[1]) / im.size
 
-        # if out of desired range, print filename
-        if max_val < desired_range[0] or max_val > desired_range[1]:
-            # print("File {} has max value {}".format(fn, max_val))
+        if max_val < desired_range[0]:
             n_bad_files += 1
             bad_files.append(fn)
 
@@ -61,6 +74,28 @@ def analyze_dataset(config):
             else:
                 print("File {} has max value {}".format(fn, max_val))
 
+        elif saturation_ratio > saturation_percent:
+            n_bad_files += 1
+            bad_files.append(fn)
+
+            if delete_bad:
+                os.remove(fn)
+                print("REMOVED file {}".format(fn))
+            else:
+                print("File {} has saturation ratio {}".format(fn, saturation_ratio))
+
+        # # if out of desired range, print filename
+        # if max_val < desired_range[0] or saturation_ratio > saturation_percent:
+        #     # print("File {} has max value {}".format(fn, max_val))
+        #     n_bad_files += 1
+        #     bad_files.append(fn)
+
+        #     if delete_bad:
+        #         os.remove(fn)
+        #         print("REMOVED file {}".format(fn))
+        #     else:
+        #         print("File {} has max value {}".format(fn, max_val))
+
     proc_time = time.time() - start_time
     print("Went through {} files in {:.2f} seconds".format(len(files), proc_time))
     print(
@@ -69,6 +104,14 @@ def analyze_dataset(config):
         )
     )
 
+    # plot histogram
+    output_folder = os.getcwd()
+    output_fp = os.path.join(output_folder, "max_vals.png")
+    plt.hist(max_vals, bins=100)
+    plt.savefig(output_fp)
+
+    print("Saved histogram to {}".format(output_fp))
+
     # command line input on whether to delete bad files
     if not delete_bad:
         response = None
@@ -80,14 +123,6 @@ def analyze_dataset(config):
         else:
             print("Not deleting bad files")
 
-    # plot histogram
-    output_folder = os.getcwd()
-    output_fp = os.path.join(output_folder, "max_vals.png")
-    plt.hist(max_vals, bins=100)
-    plt.savefig(output_fp)
-
-    print("Saved histogram to {}".format(output_fp))
-
 
 if __name__ == "__main__":
     analyze_dataset()