Raise an exception on receiving duplicate filepaths (#242)

* Raise an exception on receiving duplicate filepaths to avoid out of memory error * Update src/cleanvision/dataset/fsspec_dataset.py Co-authored-by: Jonas Mueller <[email protected]> * Update tests/test_run.py Co-authored-by: Jonas Mueller <[email protected]> --------- Co-authored-by: Jonas Mueller <[email protected]>
cleanlab · Nov 22, 2023 · d1334e0 · d1334e0
1 parent f653207
commit d1334e0
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 0 deletions.
diff --git a/src/cleanvision/dataset/fsspec_dataset.py b/src/cleanvision/dataset/fsspec_dataset.py
@@ -35,6 +35,10 @@ def __init__(
             self._filepaths = self.__get_filepaths(dataset_path)
         else:
             assert filepaths is not None
+            if len(filepaths) != len(set(filepaths)):
+                raise ValueError(
+                    "Duplicate filepaths found in the provided list, please remove these duplicates."
+                )
             self._filepaths = filepaths
             # here we assume all of the provided filepaths are from the same filesystem
             self.fs, _ = fsspec.core.url_to_fs(self._filepaths[0], **self.storage_opts)

diff --git a/tests/test_run.py b/tests/test_run.py
@@ -177,6 +177,12 @@ def test_run_imagelab_given_filepaths(generate_local_dataset, images_per_class):
     assert len(imagelab.issues) == images_per_class
 
 
+def test_raise_error_given_duplicate_filepaths():
+    filepaths = ["/home/user/file0.jpg", "/home/user/file1.jpg", "/home/user/file0.jpg"]
+    with pytest.raises(ValueError, match="Duplicate"):
+        Imagelab(filepaths=filepaths)
+
+
 def test_s3_dataset(capsys, get_example_s3_filepaths):
     imagelab = Imagelab(filepaths=get_example_s3_filepaths, storage_opts={"anon": True})
     imagelab.find_issues(