Adding support for on_error parameter in walk

fsspec/asyn.py

@@ -641,7 +641,7 @@ async def _info(self, path, **kwargs):
     async def _ls(self, path, detail=True, **kwargs):
         raise NotImplementedError
 
-    async def _walk(self, path, maxdepth=None, onerror=None, **kwargs):
+    async def _walk(self, path, maxdepth=None, on_error="raise", **kwargs):
         if maxdepth is not None and maxdepth < 1:
             raise ValueError("maxdepth must be at least 1")
 
@@ -654,8 +654,10 @@ async def _walk(self, path, maxdepth=None, onerror=None, **kwargs):
         try:
             listing = await self._ls(path, detail=True, **kwargs)
         except (FileNotFoundError, OSError) as e:
-            if onerror is not None:
-                onerror(e)
+            if on_error == "raise":
+                raise
+            elif callable(on_error):
+                on_error(e)
             if detail:
                 yield path, {}, {}
             else:

fsspec/spec.py

@@ -372,7 +372,7 @@ def _ls_from_cache(self, path):
         except KeyError:
             pass
 
-    def walk(self, path, maxdepth=None, topdown=True, onerror=None, **kwargs):
+    def walk(self, path, maxdepth=None, topdown=True, on_error="raise", **kwargs):
         """Return all files belows path
 
         List all files, recursing into subdirectories; output is iterator-style,
@@ -386,11 +386,6 @@ def walk(self, path, maxdepth=None, topdown=True, onerror=None, **kwargs):
         it resumes walk() again.
         Modifying dirnames when topdown is False has no effect. (see os.walk)
 
-        If optional argument onerror is specified, it should be a function;
-        it will be called with one argument, an OSError instance.
-        It can report the error to continue with the walk,
-        or raise the exception to abort the walk.
-
         Note that the "files" outputted will include anything that is not
         a directory, such as links.
 
@@ -404,6 +399,10 @@ def walk(self, path, maxdepth=None, topdown=True, onerror=None, **kwargs):
         topdown: bool (True)
             Whether to walk the directory tree from the top downwards or from
             the bottom upwards.
+        on_error: "raise", "omit", a collable
+            If raise (default), an underlying exception will be raised;
+            if omit, path with exception will simply be empty (no dirs, no files);
+            if callable, it will be called with a single OSError instance as argument
         kwargs: passed to ``ls``
         """
         if maxdepth is not None and maxdepth < 1:
@@ -418,8 +417,10 @@ def walk(self, path, maxdepth=None, topdown=True, onerror=None, **kwargs):
         try:
             listing = self.ls(path, detail=True, **kwargs)
         except (FileNotFoundError, OSError) as e:
-            if onerror is not None:
-                onerror(e)
+            if on_error == "raise":
+                raise
+            elif callable(on_error):
+                on_error(e)
             if detail:
                 return path, {}, {}
             return path, [], []

fsspec/tests/test_api.py

@@ -481,19 +481,18 @@ def _walk(*args, **kwargs):
         (dir1, ["dir11", "dir12"], ["file11"]),
     ]
 
-    # onerror skip by default
-    assert list(m.walk("do_not_exist")) == []
-    # onerror skip function
+    # on_error raise by default
+    with pytest.raises(FileNotFoundError):
+        list(m.walk("do_not_exist"))
+    # on_error raise
+    with pytest.raises(FileNotFoundError):
+        list(m.walk("do_not_exist", on_error="raise"))
+    # on_error omit
+    assert list(m.walk("do_not_exist", on_error="omit")) == []
+    # on_error callable function
     mock = Mock()
-    assert list(m.walk("do_not_exist", onerror=mock.onerror)) == []
+    assert list(m.walk("do_not_exist", on_error=mock.onerror)) == []
     mock.onerror.assert_called()
     assert mock.onerror.call_args.kwargs == {}
     assert len(mock.onerror.call_args.args) == 1
     assert isinstance(mock.onerror.call_args.args[0], FileNotFoundError)
-    # onerror re-raise function
-    with pytest.raises(FileNotFoundError):
-
-        def onerror(e):
-            raise e
-
-        list(m.walk("do_not_exist", onerror=onerror))