From 0d7f03c4cb16fa39c06e73c35f7f3d9d942f66c7 Mon Sep 17 00:00:00 2001 From: Yomguithereal Date: Wed, 20 Dec 2023 16:41:02 +0100 Subject: [PATCH] Adding timeout kwarg to crawl_command Fix #919 --- minet/cli/crawl/__init__.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/minet/cli/crawl/__init__.py b/minet/cli/crawl/__init__.py index 93c2a53f13..ee7337c83d 100644 --- a/minet/cli/crawl/__init__.py +++ b/minet/cli/crawl/__init__.py @@ -191,6 +191,7 @@ def crawl_command( default_retries: Optional[int] = None, force_spoof_user_agent: Optional[bool] = None, force_stateful_redirects: Optional[bool] = None, + timeout: bool = True, default_timeout: Optional[float] = None, default_connect_timeout: Optional[float] = None, ): @@ -241,11 +242,16 @@ def set_default_value(k, v): if default_retries is not None: set_default_value("retries", default_retries) - if default_timeout is not None: - set_default_value("timeout", default_timeout) + if not timeout: + del arguments_dict["timeout"] + del arguments_dict["connect_timeout"] - if default_connect_timeout is not None: - set_default_value("connect_timeout", default_connect_timeout) + else: + if default_timeout is not None: + set_default_value("timeout", default_timeout) + + if default_connect_timeout is not None: + set_default_value("connect_timeout", default_connect_timeout) if force_folder_strategy is not None: del arguments_dict["folder_strategy"] @@ -337,9 +343,10 @@ def wrapped_resolve(cli_args): if resolve is not None: resolve(cli_args) - cli_args.timeout = Timeout( - connect=cli_args.connect_timeout, total=cli_args.timeout - ) + if timeout: + cli_args.timeout = Timeout( + connect=cli_args.connect_timeout, total=cli_args.timeout + ) # --sqlar disables --compress-on-disk if cli_args.sqlar: