Better error message when Spider.process returns wrong type

Fix #941
medialab · Apr 15, 2024 · 2971c4f · 2971c4f
1 parent 752a4d7
commit 2971c4f
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 10 deletions.
diff --git a/minet/crawl/crawler.py b/minet/crawl/crawler.py
@@ -204,7 +204,13 @@ def __call__(
             spider_result = spider.process(job, response)
 
             if spider_result is not None:
-                data, next_jobs = spider_result
+                try:
+                    data, next_jobs = spider_result
+                except (ValueError, TypeError):
+                    raise TypeError(
+                        'Spider.process is expected to return either None or a 2-tuple containing data and next targets to enqueue. Got a "%s" instead.'
+                        % spider_result.__class__.__name__
+                    )
             else:
                 data = None
                 next_jobs = None
@@ -505,12 +511,12 @@ def crawl(
                 "Crawler",
                 SuccessfulCrawlResult[CrawlJobDataTypes, CrawlResultDataTypes],
             ],
-            CallbackResultType,
-        ] = ...,
+            Optional[CallbackResultType],
+        ],
     ) -> Iterator[
         Tuple[
             AnyCrawlResult[CrawlJobDataTypes, CrawlResultDataTypes],
-            CallbackResultType,
+            Optional[CallbackResultType],
         ]
     ]: ...
 
@@ -522,15 +528,15 @@ def crawl(
                     "Crawler",
                     SuccessfulCrawlResult[CrawlJobDataTypes, CrawlResultDataTypes],
                 ],
-                CallbackResultType,
+                Optional[CallbackResultType],
             ]
         ] = None,
     ) -> Union[
         Iterator[AnyCrawlResult[CrawlJobDataTypes, CrawlResultDataTypes]],
         Iterator[
             Tuple[
                 AnyCrawlResult[CrawlJobDataTypes, CrawlResultDataTypes],
-                CallbackResultType,
+                Optional[CallbackResultType],
             ]
         ],
     ]:

diff --git a/minet/executors.py b/minet/executors.py
@@ -566,7 +566,9 @@ def request(
         passthrough: Literal[False] = ...,
         callback: Callable[[ItemType, str, Response], Optional[CallbackResultType]],
         **kwargs: Unpack[ExecutorRequestKwargs[ItemType]],
-    ) -> Iterator[Tuple[AnyActualRequestResult[ItemType], Optional[CallbackResultType]]]: ...
+    ) -> Iterator[
+        Tuple[AnyActualRequestResult[ItemType], Optional[CallbackResultType]]
+    ]: ...
 
     @overload
     def request(
@@ -660,17 +662,23 @@ def resolve(
         self,
         iterator: Iterable[ItemType],
         *,
-        callback: Callable[[ItemType, str, RedirectionStack], Optional[CallbackResultType]],
+        callback: Callable[
+            [ItemType, str, RedirectionStack], Optional[CallbackResultType]
+        ],
         passthrough: Literal[False] = ...,
         **kwargs: Unpack[ExecutorResolveKwargs[ItemType]],
-    ) -> Iterator[Tuple[AnyActualResolveResult[ItemType], Optional[CallbackResultType]]]: ...
+    ) -> Iterator[
+        Tuple[AnyActualResolveResult[ItemType], Optional[CallbackResultType]]
+    ]: ...
 
     @overload
     def resolve(
         self,
         iterator: Iterable[ItemType],
         *,
-        callback: Callable[[ItemType, str, RedirectionStack], Optional[CallbackResultType]],
+        callback: Callable[
+            [ItemType, str, RedirectionStack], Optional[CallbackResultType]
+        ],
         passthrough: Literal[True] = ...,
         **kwargs: Unpack[ExecutorResolveKwargs[ItemType]],
     ) -> Iterator[Tuple[AnyResolveResult[ItemType], Optional[CallbackResultType]]]: ...