Adding support for onerror parameter in walk

fsspec/asyn.py

@@ -641,7 +641,7 @@ async def _info(self, path, **kwargs):
     async def _ls(self, path, detail=True, **kwargs):
         raise NotImplementedError
 
-    async def _walk(self, path, maxdepth=None, **kwargs):
+    async def _walk(self, path, maxdepth=None, onerror=None, **kwargs):
         if maxdepth is not None and maxdepth < 1:
             raise ValueError("maxdepth must be at least 1")
 
@@ -653,7 +653,9 @@ async def _walk(self, path, maxdepth=None, **kwargs):
         detail = kwargs.pop("detail", False)
         try:
             listing = await self._ls(path, detail=True, **kwargs)
-        except (FileNotFoundError, OSError):
+        except (FileNotFoundError, OSError) as e:
+            if onerror is not None:
+                onerror(e)
             if detail:
                 yield path, {}, {}
             else:

fsspec/spec.py

@@ -372,7 +372,7 @@ def _ls_from_cache(self, path):
         except KeyError:
             pass
 
-    def walk(self, path, maxdepth=None, topdown=True, **kwargs):
+    def walk(self, path, maxdepth=None, topdown=True, onerror=None, **kwargs):
         """Return all files belows path
 
         List all files, recursing into subdirectories; output is iterator-style,
@@ -386,6 +386,11 @@ def walk(self, path, maxdepth=None, topdown=True, **kwargs):
         it resumes walk() again.
         Modifying dirnames when topdown is False has no effect. (see os.walk)
 
+        If optional argument onerror is specified, it should be a function;
+        it will be called with one argument, an OSError instance.
+        It can report the error to continue with the walk,
+        or raise the exception to abort the walk.
+
         Note that the "files" outputted will include anything that is not
         a directory, such as links.
 
@@ -412,7 +417,9 @@ def walk(self, path, maxdepth=None, topdown=True, **kwargs):
         detail = kwargs.pop("detail", False)
         try:
             listing = self.ls(path, detail=True, **kwargs)
-        except (FileNotFoundError, OSError):
+        except (FileNotFoundError, OSError) as e:
+            if onerror is not None:
+                onerror(e)
             if detail:
                 return path, {}, {}
             return path, [], []

fsspec/tests/test_api.py

@@ -4,6 +4,7 @@
 import os
 import pickle
 import tempfile
+from unittest.mock import Mock
 
 import pytest
 
@@ -479,3 +480,20 @@ def _walk(*args, **kwargs):
         (dir12, [], ["file121"]),
         (dir1, ["dir11", "dir12"], ["file11"]),
     ]
+
+    # onerror skip by default
+    assert list(m.walk("do_not_exist")) == []
+    # onerror skip function
+    mock = Mock()
+    assert list(m.walk("do_not_exist", onerror=mock.onerror)) == []
+    mock.onerror.assert_called()
+    assert mock.onerror.call_args.kwargs == {}
+    assert len(mock.onerror.call_args.args) == 1
+    assert isinstance(mock.onerror.call_args.args[0], FileNotFoundError)
+    # onerror re-raise function
+    with pytest.raises(FileNotFoundError):
+
+        def onerror(e):
+            raise e
+
+        list(m.walk("do_not_exist", onerror=onerror))