In certain scenarios, niquests timeout fails to respect its configured value #183
Comments
|
I understand how confusing this looks. Both Here is a live example that reproduce what you experienced. import logging
import niquests
import asyncio
import time
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
explain_handler = logging.StreamHandler()
explain_handler.setFormatter(
logging.Formatter("%(asctime)s | %(levelname)s | %(message)s")
)
logger.addHandler(explain_handler)
async def request(url):
start = time.perf_counter()
response = None
msg = None
try:
async with niquests.AsyncSession(resolver="in-memory://default?hosts=pie.dev:240.0.0.0&hosts=pie.dev:240.0.0.1&hosts=httpbin.org:240.0.0.0") as session:
response = await asyncio.wait_for(session.get(url, timeout=5), timeout=10)
msg = "Success"
response.raise_for_status()
except asyncio.TimeoutError:
msg = "Async Timeout Error"
except niquests.RequestException as niquests_err:
msg = f"Niquests Error | {niquests_err}"
except Exception as err:
msg = f"Unexpected Error | {err}"
finally:
end = time.perf_counter()
print(f"{msg} | {url} | Elapsed time: {round(end - start, 2)}")
return response
async def run():
tasks = [
request("https://httpbin.org"),
request("https://pie.dev"),
]
await asyncio.gather(*tasks)
def main() -> None:
asyncio.run(run())
if __name__ == "__main__":
main()It will output: Why? We purposely set TWO (fake) ip attached to pie.dev and a single one for httpbin.org. How to solve this? Set Then, the output will be as follow: It a know pain from Requests era. And we had to keep it that way by default for now. Hope that clarify, |
|
I tested with happy_eyeballs=True but the results still seem to be the same. 2024-11-22 15:28:17,572 | DEBUG | Using selector: EpollSelector
2024-11-22 15:28:17,573 | DEBUG | Converted retries value: 0 -> Retry(total=0, connect=None, read=None, redirect=None, status=None)
2024-11-22 15:28:17,573 | DEBUG | Converted retries value: 0 -> Retry(total=0, connect=None, read=None, redirect=None, status=None)
2024-11-22 15:28:17,576 | DEBUG | Starting new HTTPS connection (1): httpbin.org:443
2024-11-22 15:28:17,576 | DEBUG | Attempting Happy-Eyeball httpbin.org:443
2024-11-22 15:28:17,576 | DEBUG | Converted retries value: 0 -> Retry(total=0, connect=None, read=None, redirect=None, status=None)
2024-11-22 15:28:17,576 | DEBUG | Converted retries value: 0 -> Retry(total=0, connect=None, read=None, redirect=None, status=None)
2024-11-22 15:28:17,578 | DEBUG | Starting new HTTPS connection (1): pie.dev:443
2024-11-22 15:28:17,578 | DEBUG | Attempting Happy-Eyeball pie.dev:443
Niquests Error | ('Connection aborted.', gaierror('Name or service not known: pie.dev using 1 resolver(s)')) | https://pie.dev | Elapsed time: 25.03
Niquests Error | ('Connection aborted.', gaierror('Name or service not known: httpbin.org using 1 resolver(s)')) | https://httpbin.org | Elapsed time: 25.03 |
|
OK. Something seems off. Remove your asyncio.wait_for, and intentionally make the program stop using KeyboardInterupt CTRL+C. |
diff --git a/orig.py b/test.py
index 7b405b5..e2d692c 100644
--- a/orig.py
+++ b/test.py
@@ -18,11 +18,9 @@ async def request(url):
try:
async with niquests.AsyncSession() as session:
- response = await asyncio.wait_for(session.get(url, timeout=10), timeout=20)
+ response = await session.get(url, timeout=None)
msg = "Success"
response.raise_for_status()
- except asyncio.TimeoutError:
- msg = "Async Timeout Error"
except niquests.RequestException as niquests_err:
msg = f"Niquests Error | {niquests_err}"
except Exception as err:
@@ -42,7 +40,10 @@ async def run():
await asyncio.gather(*tasks)
def main() -> None:
- asyncio.run(run())
+ try:
+ asyncio.run(run())
+ except KeyboardInterrupt:
+ print("Program interrupted by user")
if __name__ == "__main__":
main()
Case 1) without 2024-11-22 16:10:53,593 | DEBUG | Using selector: EpollSelector
2024-11-22 16:10:53,594 | DEBUG | Converted retries value: 0 -> Retry(total=0, connect=None, read=None, redirect=None, status=None)
2024-11-22 16:10:53,595 | DEBUG | Converted retries value: 0 -> Retry(total=0, connect=None, read=None, redirect=None, status=None)
2024-11-22 16:10:53,596 | DEBUG | Starting new HTTPS connection (1): httpbin.org:443
2024-11-22 16:10:53,597 | DEBUG | Converted retries value: 0 -> Retry(total=0, connect=None, read=None, redirect=None, status=None)
2024-11-22 16:10:53,598 | DEBUG | Converted retries value: 0 -> Retry(total=0, connect=None, read=None, redirect=None, status=None)
2024-11-22 16:10:53,599 | DEBUG | Starting new HTTPS connection (1): pie.dev:443
Niquests Error | AsyncHTTPSConnectionPool(host='httpbin.org', port=443): Max retries exceeded with url: / (Caused by ConnectTimeoutError(<urllib3._async.connection.AsyncHTTPSConnection object at 0x7ffff5cd9670>, 'Connection to httpbin.org timed out. (connect timeout=None)')) | https://httpbin.org | Elapsed time: 268.94
Niquests Error | AsyncHTTPSConnectionPool(host='pie.dev', port=443): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3._async.connection.AsyncHTTPSConnection object at 0x7ffff5cda780>: Failed to establish a new connection: [Errno 101] Network is unreachable')) | https://pie.dev | Elapsed time: 268.94Case 2) with 2024-11-22 16:21:09,398 | DEBUG | Using selector: EpollSelector
2024-11-22 16:21:09,399 | DEBUG | Converted retries value: 0 -> Retry(total=0, connect=None, read=None, redirect=None, status=None)
2024-11-22 16:21:09,399 | DEBUG | Converted retries value: 0 -> Retry(total=0, connect=None, read=None, redirect=None, status=None)
2024-11-22 16:21:09,401 | DEBUG | Starting new HTTPS connection (1): httpbin.org:443
2024-11-22 16:21:09,401 | DEBUG | Attempting Happy-Eyeball httpbin.org:443
2024-11-22 16:21:09,401 | DEBUG | Converted retries value: 0 -> Retry(total=0, connect=None, read=None, redirect=None, status=None)
2024-11-22 16:21:09,401 | DEBUG | Converted retries value: 0 -> Retry(total=0, connect=None, read=None, redirect=None, status=None)
2024-11-22 16:21:09,405 | DEBUG | Starting new HTTPS connection (1): pie.dev:443
2024-11-22 16:21:09,405 | DEBUG | Attempting Happy-Eyeball pie.dev:443
Niquests Error | ('Connection aborted.', gaierror('Name or service not known: httpbin.org using 1 resolver(s)')) | https://httpbin.org | Elapsed time: 25.03
Niquests Error | ('Connection aborted.', gaierror('Name or service not known: pie.dev using 1 resolver(s)')) | https://pie.dev | Elapsed time: 25.03 |
|
You misunderstood. import logging
import niquests
import asyncio
import time
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
explain_handler = logging.StreamHandler()
explain_handler.setFormatter(
logging.Formatter("%(asctime)s | %(levelname)s | %(message)s")
)
logger.addHandler(explain_handler)
async def request(url):
start = time.perf_counter()
response = None
msg = None
try:
async with niquests.AsyncSession() as session:
response = await session.get(url, timeout=10)
msg = "Success"
response.raise_for_status()
except asyncio.TimeoutError:
msg = "Async Timeout Error"
except niquests.RequestException as niquests_err:
msg = f"Niquests Error | {niquests_err}"
except Exception as err:
msg = f"Unexpected Error | {err}"
finally:
end = time.perf_counter()
print(f"{msg} | {url} | Elapsed time: {round(end - start, 2)}")
return response
async def run():
tasks = [
request("https://httpbin.org"),
request("https://pie.dev"),
]
await asyncio.gather(*tasks)
def main() -> None:
asyncio.run(run())
if __name__ == "__main__":
main()And CTRL+C when it hangs. I expect something like: |
2024-11-22 16:26:23,221 | DEBUG | Using selector: EpollSelector
2024-11-22 16:26:23,222 | DEBUG | Converted retries value: 0 -> Retry(total=0, connect=None, read=None, redirect=None, status=None)
2024-11-22 16:26:23,222 | DEBUG | Converted retries value: 0 -> Retry(total=0, connect=None, read=None, redirect=None, status=None)
2024-11-22 16:26:23,224 | DEBUG | Starting new HTTPS connection (1): httpbin.org:443
2024-11-22 16:26:23,226 | DEBUG | Converted retries value: 0 -> Retry(total=0, connect=None, read=None, redirect=None, status=None)
2024-11-22 16:26:23,227 | DEBUG | Converted retries value: 0 -> Retry(total=0, connect=None, read=None, redirect=None, status=None)
2024-11-22 16:26:23,228 | DEBUG | Starting new HTTPS connection (1): pie.dev:443
^CNone | https://httpbin.org | Elapsed time: 7.28
None | https://pie.dev | Elapsed time: 7.28
Traceback (most recent call last):
File "/nix/store/px2nj16i5gc3d4mnw5l1nclfdxhry61p-python3-3.12.7/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/nix/store/px2nj16i5gc3d4mnw5l1nclfdxhry61p-python3-3.12.7/lib/python3.12/asyncio/base_events.py", line 687, in run_until_complete
return future.result()
^^^^^^^^^^^^^^^
File "/home/user/Projects/site_checker/test.py", line 42, in run
await asyncio.gather(*tasks)
File "/home/user/Projects/site_checker/test.py", line 21, in request
response = await session.get(url, timeout=10)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/nix/store/w5kzxdffspmzwi5bn2ywqxfd8fxnd3br-python3-3.12.7-env/lib/python3.12/site-packages/niquests/_async.py", line 913, in get
return await self.request( # type: ignore[call-overload,misc]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/nix/store/w5kzxdffspmzwi5bn2ywqxfd8fxnd3br-python3-3.12.7-env/lib/python3.12/site-packages/niquests/_async.py", line 856, in request
return await self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/nix/store/w5kzxdffspmzwi5bn2ywqxfd8fxnd3br-python3-3.12.7-env/lib/python3.12/site-packages/niquests/_async.py", line 482, in send
r = await adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/nix/store/w5kzxdffspmzwi5bn2ywqxfd8fxnd3br-python3-3.12.7-env/lib/python3.12/site-packages/niquests/adapters.py", line 2038, in send
resp_or_promise = await conn.urlopen( # type: ignore[call-overload,misc]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/nix/store/w5kzxdffspmzwi5bn2ywqxfd8fxnd3br-python3-3.12.7-env/lib/python3.12/site-packages/urllib3/_async/connectionpool.py", line 1709, in urlopen
response = await self._make_request( # type: ignore[call-overload,misc]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/nix/store/w5kzxdffspmzwi5bn2ywqxfd8fxnd3br-python3-3.12.7-env/lib/python3.12/site-packages/urllib3/_async/connectionpool.py", line 1216, in _make_request
await self._validate_conn(conn)
File "/nix/store/w5kzxdffspmzwi5bn2ywqxfd8fxnd3br-python3-3.12.7-env/lib/python3.12/site-packages/urllib3/_async/connectionpool.py", line 2296, in _validate_conn
await super()._validate_conn(conn)
File "/nix/store/w5kzxdffspmzwi5bn2ywqxfd8fxnd3br-python3-3.12.7-env/lib/python3.12/site-packages/urllib3/_async/connectionpool.py", line 755, in _validate_conn
await conn.connect()
File "/nix/store/w5kzxdffspmzwi5bn2ywqxfd8fxnd3br-python3-3.12.7-env/lib/python3.12/site-packages/urllib3/_async/connection.py", line 759, in connect
self.sock = sock = await self._new_conn()
^^^^^^^^^^^^^^^^^^^^^^
File "/nix/store/w5kzxdffspmzwi5bn2ywqxfd8fxnd3br-python3-3.12.7-env/lib/python3.12/site-packages/urllib3/_async/connection.py", line 211, in _new_conn
sock = await self._resolver.create_connection(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/nix/store/w5kzxdffspmzwi5bn2ywqxfd8fxnd3br-python3-3.12.7-env/lib/python3.12/site-packages/urllib3/contrib/resolver/_async/protocols.py", line 153, in create_connection
await sock.connect(sa)
File "/nix/store/w5kzxdffspmzwi5bn2ywqxfd8fxnd3br-python3-3.12.7-env/lib/python3.12/site-packages/urllib3/contrib/ssa/__init__.py", line 240, in connect
await asyncio.wait_for(
File "/nix/store/px2nj16i5gc3d4mnw5l1nclfdxhry61p-python3-3.12.7/lib/python3.12/asyncio/tasks.py", line 520, in wait_for
return await fut
^^^^^^^^^
File "/nix/store/px2nj16i5gc3d4mnw5l1nclfdxhry61p-python3-3.12.7/lib/python3.12/asyncio/selector_events.py", line 651, in sock_connect
return await fut
^^^^^^^^^
asyncio.exceptions.CancelledError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/user/Projects/site_checker/test.py", line 48, in <module>
main()
File "/home/user/Projects/site_checker/test.py", line 45, in main
asyncio.run(run())
File "/nix/store/px2nj16i5gc3d4mnw5l1nclfdxhry61p-python3-3.12.7/lib/python3.12/asyncio/runners.py", line 194, in run
return runner.run(main)
^^^^^^^^^^^^^^^^
File "/nix/store/px2nj16i5gc3d4mnw5l1nclfdxhry61p-python3-3.12.7/lib/python3.12/asyncio/runners.py", line 123, in run
raise KeyboardInterrupt()
KeyboardInterrupt |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
When the network is connected to a VPN server that is unresponsive, the niquests timeout option does not seem to respect the specified value and instead set to {timeout * 2} or more seconds or hangs indefinitely. This issue has occurred multiple times with my wireguard setup.
Expected Result
This is the result when there is no VPN connection, but the internet connection has been cut off. As expected, it immediately returned a resolution error.
Actual Result
This is the result when there is a VPN connection, but the internet connection has been cut off. Instead of throwing a resolution error or respecting the specified timeout value of 10 seconds, it hangs for the default timeout value of 20 seconds or more or hangs indefinitely. However, since the asyncio.wait_for timeout was set to 20 seconds, it ultimately raised an asyncio.TimeoutError.
Case 1) asyicio_wait_for timeout is 20s
Case 2) asyncio_wait_for timeout is longer than niquests timeout * 2
Reproduction Steps
System Information
{ "charset_normalizer": { "version": "3.4.0" }, "http1": { "h11": "0.14.0" }, "http2": { "jh2": "5.0.4" }, "http3": { "enabled": true, "qh3": "1.2.1" }, "idna": { "version": "3.10" }, "implementation": { "name": "CPython", "version": "3.12.7" }, "niquests": { "version": "3.11.1" }, "ocsp": { "enabled": true }, "platform": { "release": "6.6.62", "system": "Linux" }, "system_ssl": { "version": "30300020" }, "urllib3.future": { "cohabitation_version": null, "version": "2.11.911" }, "wassima": { "certifi_fallback": false, "enabled": true, "version": "1.1.5" } }The text was updated successfully, but these errors were encountered: