diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 0b7dc997be..8637203783 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -21,13 +21,13 @@ jobs: fail-fast: false matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "pypy-3.9", "pypy-3.10"] - os: [ubuntu-latest, macOS-13, windows-latest] + os: [ubuntu-22.04, macOS-13, windows-latest] include: # pypy-3.7, pypy-3.8 may fail due to missing cryptography wheels. Adapting. - python-version: pypy-3.7 - os: ubuntu-latest + os: ubuntu-22.04 - python-version: pypy-3.8 - os: ubuntu-latest + os: ubuntu-22.04 - python-version: pypy-3.8 os: macOS-13 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5a676282e5..764173bbe7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,4 +28,4 @@ repos: - id: mypy args: [--check-untyped-defs] exclude: 'tests/|noxfile.py' - additional_dependencies: ['charset_normalizer', 'urllib3.future>=2.10.900', 'wassima>=1.0.1', 'idna', 'kiss_headers'] + additional_dependencies: ['charset_normalizer', 'urllib3.future>=2.10.903', 'wassima>=1.0.1', 'idna', 'kiss_headers'] diff --git a/HISTORY.md b/HISTORY.md index ce9b0cafd4..b6e8c1682b 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,22 @@ Release History =============== +3.9.1 (2024-10-13) +------------------ + +**Fixed** +- Exception leak from urllib3-future when using WebSocket. +- Enforcing HTTP/3 in an AsyncSession. (#152) +- Adapter kwargs fallback to support old Requests extensions. +- Type hint for ``Response.extension`` linked to the generic interface instead of the inherited ones. +- Accessing WS over HTTP/2+ using the synchronous session object. + +**Misc** +- Documentation improvement for in-memory certificates and WebSocket use cases. + +**Changed** +- urllib3-future lower bound version is raised to 2.10.904 to ensure exception are properly translated into urllib3-future ones for WS. + 3.9.0 (2024-10-08) ------------------ diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index d8efe743f3..89f4f0b6f9 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -313,6 +313,10 @@ In-memory Certificates The ``cert=...`` and ``verify=...`` can actually take the certificates themselves. Niquests support in-memory certificates instead of file paths. +.. warning:: The mTLS (aka. ``cert=...``) using in-memory certificate only works with Linux, FreeBSD or OpenBSD. See https://urllib3future.readthedocs.io/en/latest/advanced-usage.html#in-memory-client-mtls-certificate for more. It works on all platforms if you are using HTTP/3 over QUIC. + +.. note:: When leveraging in-memory certificate for mTLS (aka. ``cert=...``), you have two possible configurations: (cert, key) or (cert, key, password) you cannot pass (cert) having concatenated cert,key in a single string. + .. _ca-certificates: CA Certificates @@ -320,7 +324,8 @@ CA Certificates Niquests uses certificates provided by the package `wassima`_. This allows for users to not care about root CAs. By default it is expected to use your operating system root CAs. -You have nothing to do. +You have nothing to do. If we were unable to access your OS truststore natively, (e.g. not Windows, not MacOS, not Linux), then +we will fallback on the ``certifi`` bundle. .. _HTTP persistent connection: https://en.wikipedia.org/wiki/HTTP_persistent_connection .. _connection pooling: https://urllib3.readthedocs.io/en/latest/reference/index.html#module-urllib3.connectionpool diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 07cb25ea61..bb488d9ef3 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -1120,6 +1120,135 @@ Others Every other features still applies with WebSocket, like proxies, happy eyeballs, thread/task safety, etc... See relevant docs for more. +Example with Concurrency (Thread) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In the following example, we will see how to communicate with a WebSocket server that echo what we send to him. +We will use a Thread for the reads and the main thread for write operations. + +See:: + + from __future__ import annotations + + from niquests import Session, Response, ReadTimeout + from threading import Thread + from time import sleep + + + def pull_message_from_server(my_response: Response) -> None: + """Read messages here.""" + iteration_counter = 0 + + while my_response.extension.closed is False: + try: + # will block for 1s top + message = my_response.extension.next_payload() + + if message is None: # server just closed the connection. exit. + print("received goaway from server") + return + + print(f"received message: '{message}'") + except ReadTimeout: # if no message received within 1s + pass + + sleep(1) # let some time for the write part to acquire the lock + iteration_counter += 1 + + # send a ping every four iteration + if iteration_counter % 4 == 0: + my_response.extension.ping() + print("ping sent") + + if __name__ == "__main__": + + with Session() as s: + # connect to websocket server "echo.websocket.org" with timeout of 1s (both read and connect) + resp = s.get("wss://echo.websocket.org", timeout=1) + + if resp.status_code != 101: + exit(1) + + t = Thread(target=pull_message_from_server, args=(resp,)) + t.start() + + # send messages here + for i in range(30): + to_send = f"Hello World {i}" + resp.extension.send_payload(to_send) + print(f"sent message: '{to_send}'") + sleep(1) # let some time for the read part to acquire the lock + + # exit gently! + resp.extension.close() + + # wait for thread proper exit. + t.join() + + print("program ended!") + + +.. warning:: The sleep serve the purpose to relax the lock on either the read or write side, so that one would not block the other forever. + +Example with Concurrency (Async) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The same example as before, but using async instead. + +See:: + + import asyncio + from niquests import AsyncSession, ReadTimeout, Response + + async def read_from_ws(my_response: Response) -> None: + iteration_counter = 0 + + while my_response.extension.closed is False: + try: + # will block for 1s top + message = await my_response.extension.next_payload() + + if message is None: # server just closed the connection. exit. + print("received goaway from server") + return + + print(f"received message: '{message}'") + except ReadTimeout: # if no message received within 1s + pass + + await asyncio.sleep(1) # let some time for the write part to acquire the lock + iteration_counter += 1 + + # send a ping every four iteration + if iteration_counter % 4 == 0: + await my_response.extension.ping() + print("ping sent") + + async def main() -> None: + async with AsyncSession() as s: + resp = await s.get("wss://echo.websocket.org", timeout=1) + + print(resp) + + task = asyncio.create_task(read_from_ws(resp)) + + for i in range(30): + to_send = f"Hello World {i}" + await resp.extension.send_payload(to_send) + print(f"sent message: '{to_send}'") + await asyncio.sleep(1) # let some time for the read part to acquire the lock + + # exit gently! + await resp.extension.close() + await task + + + if __name__ == "__main__": + asyncio.run(main()) + + +.. note:: The given example are really basic ones. You may adjust at will the settings and algorithm to match your requisites. + ----------------------- Ready for more? Check out the :ref:`advanced ` section. diff --git a/pyproject.toml b/pyproject.toml index 9baf880044..339609fcaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dynamic = ["version"] dependencies = [ "charset_normalizer>=2,<4", "idna>=2.5,<4", - "urllib3.future>=2.10.900,<3", + "urllib3.future>=2.10.904,<3", "wassima>=1.0.1,<2", "kiss_headers>=2,<4", ] diff --git a/src/niquests/__version__.py b/src/niquests/__version__.py index 6d9274892a..399e2182f9 100644 --- a/src/niquests/__version__.py +++ b/src/niquests/__version__.py @@ -9,9 +9,9 @@ __url__: str = "https://niquests.readthedocs.io" __version__: str -__version__ = "3.9.0" +__version__ = "3.9.1" -__build__: int = 0x030900 +__build__: int = 0x030901 __author__: str = "Kenneth Reitz" __author_email__: str = "me@kennethreitz.org" __license__: str = "Apache-2.0" diff --git a/src/niquests/_async.py b/src/niquests/_async.py index 316420266c..a25ffca34e 100644 --- a/src/niquests/_async.py +++ b/src/niquests/_async.py @@ -243,6 +243,7 @@ def __init__( AsyncHTTPAdapter( quic_cache_layer=self.quic_cache_layer, max_retries=retries, + disable_http1=disable_http1, disable_http2=disable_http2, disable_http3=disable_http3, resolver=resolver, @@ -258,6 +259,9 @@ def __init__( "http://", AsyncHTTPAdapter( max_retries=retries, + disable_http1=disable_http1, + disable_http2=disable_http2, + disable_http3=disable_http3, resolver=resolver, source_address=source_address, disable_ipv4=disable_ipv4, @@ -422,6 +426,7 @@ async def on_early_response(early_response: Response) -> None: AsyncHTTPAdapter( quic_cache_layer=self.quic_cache_layer, max_retries=self.retries, + disable_http1=self._disable_http1, disable_http2=self._disable_http2, disable_http3=self._disable_http3, resolver=self.resolver, @@ -437,6 +442,9 @@ async def on_early_response(early_response: Response) -> None: "http://", AsyncHTTPAdapter( max_retries=self.retries, + disable_http1=self._disable_http1, + disable_http2=self._disable_http2, + disable_http3=self._disable_http3, resolver=self.resolver, source_address=self.source_address, disable_ipv4=self._disable_ipv4, diff --git a/src/niquests/adapters.py b/src/niquests/adapters.py index a92e64c8fd..42c0c3249c 100644 --- a/src/niquests/adapters.py +++ b/src/niquests/adapters.py @@ -161,6 +161,8 @@ _deepcopy_ci, parse_scheme, is_ocsp_capable, + wrap_extension_for_http, + async_wrap_extension_for_http, ) try: @@ -911,7 +913,14 @@ def send( extension = None if scheme is not None and scheme not in ("http", "https"): - extension = load_extension(scheme)() + if "+" in scheme: + scheme, implementation = tuple(scheme.split("+", maxsplit=1)) + else: + implementation = None + + extension = wrap_extension_for_http( + load_extension(scheme, implementation=implementation) + )() def early_response_hook(early_response: BaseHTTPResponse) -> None: nonlocal on_early_response @@ -1916,7 +1925,9 @@ async def send( else: implementation = None - extension = async_load_extension(scheme, implementation=implementation)() + extension = async_wrap_extension_for_http( + async_load_extension(scheme, implementation=implementation) + )() async def early_response_hook(early_response: BaseAsyncHTTPResponse) -> None: nonlocal on_early_response diff --git a/src/niquests/models.py b/src/niquests/models.py index 9e8074e427..d31ef791dd 100644 --- a/src/niquests/models.py +++ b/src/niquests/models.py @@ -54,8 +54,14 @@ from urllib3.fields import RequestField from urllib3.filepost import choose_boundary, encode_multipart_formdata from urllib3.util import parse_url - from urllib3.contrib.webextensions._async import AsyncExtensionFromHTTP - from urllib3.contrib.webextensions import ExtensionFromHTTP + from urllib3.contrib.webextensions._async import ( + AsyncWebSocketExtensionFromHTTP, + AsyncRawExtensionFromHTTP, + ) + from urllib3.contrib.webextensions import ( + WebSocketExtensionFromHTTP, + RawExtensionFromHTTP, + ) else: from urllib3_future import ( # type: ignore[assignment] BaseHTTPResponse, @@ -73,8 +79,14 @@ from urllib3_future.fields import RequestField # type: ignore[assignment] from urllib3_future.filepost import choose_boundary, encode_multipart_formdata # type: ignore[assignment] from urllib3_future.util import parse_url # type: ignore[assignment] - from urllib3_future.contrib.webextensions._async import AsyncExtensionFromHTTP # type: ignore[assignment] - from urllib3_future.contrib.webextensions import ExtensionFromHTTP # type: ignore[assignment] + from urllib3_future.contrib.webextensions._async import ( # type: ignore[assignment] + AsyncWebSocketExtensionFromHTTP, + AsyncRawExtensionFromHTTP, + ) + from urllib3_future.contrib.webextensions import ( # type: ignore[assignment] + WebSocketExtensionFromHTTP, + RawExtensionFromHTTP, + ) from ._typing import ( BodyFormType, @@ -1023,10 +1035,21 @@ def __init__(self) -> None: self.download_progress: TransferProgress | None = None @property - def extension(self) -> ExtensionFromHTTP | None: - """Access the I/O after an Upgraded connection. E.g. for a WebSocket handler.""" + def extension( + self, + ) -> ( + WebSocketExtensionFromHTTP + | RawExtensionFromHTTP + | AsyncWebSocketExtensionFromHTTP + | AsyncRawExtensionFromHTTP + | None + ): + """Access the I/O after an Upgraded connection. E.g. for a WebSocket handler. + If the server opened a WebSocket, then the extension will be of type WebSocketExtensionFromHTTP. + Otherwise, on unknown protocol, it will be RawExtensionFromHTTP. + Warning: If you stand in an async inclosure, the type will be AsyncWebSocketExtensionFromHTTP or AsyncRawExtensionFromHTTP.""" return ( - self.raw.extension + self.raw.extension # type: ignore[return-value] if self.raw is not None and hasattr(self.raw, "extension") else None ) @@ -1612,10 +1635,14 @@ class AsyncResponse(Response): } @property - def extension(self) -> AsyncExtensionFromHTTP | None: # type: ignore[override] - """Access the I/O after an Upgraded connection. E.g. for a WebSocket handler.""" + def extension( # type: ignore[override] + self, + ) -> AsyncWebSocketExtensionFromHTTP | AsyncRawExtensionFromHTTP | None: + """Access the I/O after an Upgraded connection. E.g. for a WebSocket handler. + If the server opened a WebSocket, then the extension will be of type AsyncWebSocketExtensionFromHTTP. + Otherwise, on unknown protocol, it will be AsyncRawExtensionFromHTTP.""" return ( - self.raw.extension + self.raw.extension # type: ignore[return-value] if self.raw is not None and hasattr(self.raw, "extension") else None ) diff --git a/src/niquests/sessions.py b/src/niquests/sessions.py index 73a9382a05..02772b5f68 100644 --- a/src/niquests/sessions.py +++ b/src/niquests/sessions.py @@ -1186,6 +1186,7 @@ def on_early_response(early_response) -> None: max_retries=self.retries, disable_http1=self._disable_http1, disable_http2=self._disable_http2, + disable_http3=self._disable_http3, resolver=self.resolver, source_address=self.source_address, disable_ipv4=self._disable_ipv4, @@ -1222,6 +1223,7 @@ def on_early_response(early_response) -> None: del kwargs["multiplexed"] del kwargs["on_upload_body"] del kwargs["on_post_connection"] + del kwargs["on_early_response"] r = adapter.send(request, **kwargs) diff --git a/src/niquests/utils.py b/src/niquests/utils.py index fe61aad5c4..d95863fce6 100644 --- a/src/niquests/utils.py +++ b/src/niquests/utils.py @@ -50,7 +50,8 @@ AsyncManyResolver, ) from urllib3 import ConnectionInfo - from urllib3.contrib.webextensions import load_extension + from urllib3.contrib.webextensions import load_extension, ExtensionFromHTTP + from urllib3.contrib.webextensions._async import AsyncExtensionFromHTTP else: from urllib3_future.util import make_headers, parse_url # type: ignore[assignment] from urllib3_future.contrib.resolver import ( # type: ignore[assignment] @@ -65,7 +66,8 @@ AsyncManyResolver, ) from urllib3_future import ConnectionInfo # type: ignore[assignment] - from urllib3_future.contrib.webextensions import load_extension # type: ignore[assignment] + from urllib3_future.contrib.webextensions import load_extension, ExtensionFromHTTP # type: ignore[assignment] + from urllib3_future.contrib.webextensions._async import AsyncExtensionFromHTTP # type: ignore[assignment] from .__version__ import __version__ from .exceptions import InvalidURL, UnrewindableBodyError, MissingSchema @@ -1237,3 +1239,193 @@ def is_ocsp_capable(conn_info: ConnectionInfo | None) -> bool: return False return True + + +def wrap_extension_for_http( + extension: type[ExtensionFromHTTP], +) -> type[ExtensionFromHTTP]: + """ + We want to properly map exceptions from bellow (urllib3-future) into our own exceptions. + This function purposely wrap the extension class to achieve that. + Warning: synchronous context only! + """ + if HAS_LEGACY_URLLIB3 is False: + from urllib3.exceptions import ( + ClosedPoolError, + HTTPError as _HTTPError, + InvalidHeader as _InvalidHeader, + ProtocolError, + ProxyError as _ProxyError, + ReadTimeoutError, + SSLError as _SSLError, + DecodeError, + ) + else: + from urllib3_future.exceptions import ( # type: ignore[assignment] + ClosedPoolError, + HTTPError as _HTTPError, + InvalidHeader as _InvalidHeader, + ProtocolError, + ProxyError as _ProxyError, + ReadTimeoutError, + SSLError as _SSLError, + DecodeError, + ) + + from .exceptions import ( + ConnectionError, + InvalidHeader, + ProxyError, + ReadTimeout, + SSLError as RequestsSSLError, + ContentDecodingError, + ChunkedEncodingError, + ) + + class _WrappedExtensionFromHTTP(extension): # type: ignore[valid-type,misc] + def next_payload(self) -> str | bytes | None: + try: + return super().next_payload() + except ProtocolError as e: + raise ChunkedEncodingError(e) + except DecodeError as e: + raise ContentDecodingError(e) + except ReadTimeoutError as e: + raise ReadTimeout(e) + except _SSLError as e: + raise RequestsSSLError(e) + + def send_payload(self, buf: str | bytes) -> None: + try: + super().send_payload(buf) + except (ProtocolError, OSError) as err: + raise ConnectionError(err) + except ClosedPoolError as e: + raise ConnectionError(e) + except _ProxyError as e: + raise ProxyError(e) + except (_SSLError, _HTTPError) as e: + if isinstance(e, _SSLError): + raise RequestsSSLError(e) + elif isinstance(e, ReadTimeoutError): + raise ReadTimeout(e) + elif isinstance(e, _InvalidHeader): + raise InvalidHeader(e) + else: + raise + + def close(self) -> None: + try: + super().close() + except (ProtocolError, OSError) as err: + raise ConnectionError(err) + except ClosedPoolError as e: + raise ConnectionError(e) + except _ProxyError as e: + raise ProxyError(e) + except (_SSLError, _HTTPError) as e: + if isinstance(e, _SSLError): + raise RequestsSSLError(e) + elif isinstance(e, ReadTimeoutError): + raise ReadTimeout(e) + elif isinstance(e, _InvalidHeader): + raise InvalidHeader(e) + else: + raise + + return _WrappedExtensionFromHTTP + + +def async_wrap_extension_for_http( + extension: type[AsyncExtensionFromHTTP], +) -> type[AsyncExtensionFromHTTP]: + """ + We want to properly map exceptions from bellow (urllib3-future) into our own exceptions. + This function purposely wrap the extension class to achieve that. + Warning: asynchronous context only! + """ + if HAS_LEGACY_URLLIB3 is False: + from urllib3.exceptions import ( + ClosedPoolError, + HTTPError as _HTTPError, + InvalidHeader as _InvalidHeader, + ProtocolError, + ProxyError as _ProxyError, + ReadTimeoutError, + SSLError as _SSLError, + DecodeError, + ) + else: + from urllib3_future.exceptions import ( # type: ignore[assignment] + ClosedPoolError, + HTTPError as _HTTPError, + InvalidHeader as _InvalidHeader, + ProtocolError, + ProxyError as _ProxyError, + ReadTimeoutError, + SSLError as _SSLError, + DecodeError, + ) + + from .exceptions import ( + ConnectionError, + InvalidHeader, + ProxyError, + ReadTimeout, + SSLError as RequestsSSLError, + ContentDecodingError, + ChunkedEncodingError, + ) + + class _AsyncWrappedExtensionFromHTTP(extension): # type: ignore[valid-type,misc] + async def next_payload(self) -> str | bytes | None: + try: + return await super().next_payload() + except ProtocolError as e: + raise ChunkedEncodingError(e) + except DecodeError as e: + raise ContentDecodingError(e) + except ReadTimeoutError as e: + raise ReadTimeout(e) + except _SSLError as e: + raise RequestsSSLError(e) + + async def send_payload(self, buf: str | bytes) -> None: + try: + await super().send_payload(buf) + except (ProtocolError, OSError) as err: + raise ConnectionError(err) + except ClosedPoolError as e: + raise ConnectionError(e) + except _ProxyError as e: + raise ProxyError(e) + except (_SSLError, _HTTPError) as e: + if isinstance(e, _SSLError): + raise RequestsSSLError(e) + elif isinstance(e, ReadTimeoutError): + raise ReadTimeout(e) + elif isinstance(e, _InvalidHeader): + raise InvalidHeader(e) + else: + raise + + async def close(self) -> None: + try: + await super().close() + except (ProtocolError, OSError) as err: + raise ConnectionError(err) + except ClosedPoolError as e: + raise ConnectionError(e) + except _ProxyError as e: + raise ProxyError(e) + except (_SSLError, _HTTPError) as e: + if isinstance(e, _SSLError): + raise RequestsSSLError(e) + elif isinstance(e, ReadTimeoutError): + raise ReadTimeout(e) + elif isinstance(e, _InvalidHeader): + raise InvalidHeader(e) + else: + raise + + return _AsyncWrappedExtensionFromHTTP