Fix crawl_command interaction of url_cache & unique kwargs

Fix #924
medialab · Dec 20, 2023 · 34b407e · 34b407e
1 parent 8dd148d
commit 34b407e
Showing 1 changed file with 9 additions and 3 deletions.
diff --git a/minet/cli/crawl/__init__.py b/minet/cli/crawl/__init__.py
@@ -158,6 +158,12 @@
     },
 ]
 
+def delete(o, k):
+    try:
+        del o[k]
+    except KeyError:
+        pass
+
 
 def crawl_command(
     name: str,
@@ -198,11 +204,11 @@ def set_default_value(k, v):
 
     # NOTE: missing a lot in the resolve here
     if unique:
-        del arguments_dict["visit_urls_only_once"]
+        delete(arguments_dict, "visit_urls_only_once")
 
     if not url_cache:
-        del arguments_dict["visit_urls_only_once"]
-        del arguments_dict["normalized_url_cache"]
+        delete(arguments_dict, "visit_urls_only_once")
+        delete(arguments_dict, "normalized_url_cache")
 
     if not max_depth:
         del arguments_dict["max_depth"]