From 59f89e98253339a372e2889e51a2211b3752b55b Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 1 Nov 2023 22:05:34 +0100 Subject: [PATCH] :bookmark: Release 2.2.900 See CHANGES.rst for more details --- .github/FUNDING.yml | 5 +- .github/ISSUE_TEMPLATE/config.yml | 5 +- .github/PULL_REQUEST_TEMPLATE.md | 6 +- .github/PULL_REQUEST_TEMPLATE/release.md | 15 +- .github/workflows/ci.yml | 6 +- .github/workflows/integration.yml | 2 +- .pre-commit-config.yaml | 2 +- CHANGES.rst | 64 +++ README.md | 27 +- dev-requirements.txt | 1 - docs/advanced-usage.rst | 96 +++++ docs/conf.py | 11 +- docs/index.rst | 11 +- docs/reference/urllib3.backend.rst | 6 + docs/reference/urllib3.connection.rst | 2 +- docs/v2-migration-guide.rst | 46 +-- dummyserver/testcase.py | 2 +- noxfile.py | 13 - pyproject.toml | 8 +- setup.cfg | 2 +- src/urllib3/__init__.py | 16 +- src/urllib3/_constant.py | 226 +++++++++++ src/urllib3/_request_methods.py | 147 ++++++- src/urllib3/_version.py | 2 +- src/urllib3/backend/__init__.py | 2 + src/urllib3/backend/_base.py | 110 ++++- src/urllib3/backend/hface.py | 174 +++++--- src/urllib3/connection.py | 125 +++--- src/urllib3/connectionpool.py | 375 ++++++++++++++++-- src/urllib3/contrib/hface/_configuration.py | 4 +- .../contrib/hface/protocols/_protocols.py | 29 +- .../contrib/hface/protocols/http1/_h11.py | 5 +- .../contrib/hface/protocols/http2/_h2.py | 6 +- .../contrib/hface/protocols/http3/_qh3.py | 36 +- src/urllib3/contrib/imcc/__init__.py | 117 ++++++ src/urllib3/contrib/pyopenssl.py | 6 +- src/urllib3/exceptions.py | 19 +- src/urllib3/poolmanager.py | 272 ++++++++++++- src/urllib3/response.py | 37 +- src/urllib3/util/retry.py | 12 +- src/urllib3/util/ssl_.py | 67 +++- src/urllib3/util/ssltransport.py | 58 +-- test/test_connection.py | 3 +- test/test_connectionpool.py | 5 +- test/test_poolmanager.py | 16 +- test/test_response.py | 11 +- test/test_ssl.py | 3 +- test/test_ssltransport.py | 93 ----- test/with_dummyserver/test_connection.py | 2 +- test/with_dummyserver/test_connectionpool.py | 11 +- test/with_dummyserver/test_https.py | 35 ++ test/with_traefik/test_connection.py | 3 +- .../test_connection_multiplexed.py | 99 +++++ .../test_connectionpool_multiplexed.py | 45 +++ test/with_traefik/test_send_data.py | 8 +- test/with_traefik/test_stream.py | 4 +- 56 files changed, 2015 insertions(+), 498 deletions(-) create mode 100644 src/urllib3/_constant.py create mode 100644 src/urllib3/contrib/imcc/__init__.py create mode 100644 test/with_traefik/test_connection_multiplexed.py create mode 100644 test/with_traefik/test_connectionpool_multiplexed.py diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 5e16c95532..ec8758c709 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,3 +1,2 @@ -tidelift: pypi/urllib3 -github: urllib3 -open_collective: urllib3 +github: + - Ousret diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 6cbf5a88f1..52eb7ef64c 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,11 +1,8 @@ blank_issues_enabled: false contact_links: - name: 📚 Documentation - url: https://urllib3.readthedocs.io + url: https://urllib3future.readthedocs.io about: Make sure you read the relevant docs - name: ❓ Ask on StackOverflow url: https://stackoverflow.com/questions/tagged/urllib3 about: Ask questions about usage in StackOverflow - - name: 💬 Ask the Community - url: https://discord.gg/CHEgCZN - about: Join urllib3's Discord server diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index e5765d5108..fa19d07016 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,8 +1,8 @@ diff --git a/.github/PULL_REQUEST_TEMPLATE/release.md b/.github/PULL_REQUEST_TEMPLATE/release.md index 5dc84b28dc..0ab649f716 100644 --- a/.github/PULL_REQUEST_TEMPLATE/release.md +++ b/.github/PULL_REQUEST_TEMPLATE/release.md @@ -1,19 +1,8 @@ * [ ] See if all tests, including integration, pass -* [ ] Get the release pull request approved by a [CODEOWNER](https://github.com/urllib3/urllib3/blob/main/.github/CODEOWNERS) +* [ ] Get the release pull request approved by a [CODEOWNER](https://github.com/jawah/urllib3.future/blob/main/.github/CODEOWNERS) * [ ] Squash merge the release pull request with message "`Release `" -* [ ] Tag with X.Y.Z, push tag on urllib3/urllib3 (not on your fork, update `` accordingly) - * Notice that the `` shouldn't have a `v` prefix (Use `1.26.6` instead of `v.1.26.6`) - * ``` - git tag -s -a '' -m 'Release: ' - git push --tags - ``` -* [ ] Execute the `publish` GitHub workflow. This requires a review from a maintainer. +* [ ] Create a Github Release * [ ] Ensure that all expected artifacts are added to the new GitHub release. Should be one `.whl`, one `.tar.gz`, and one `multiple.intoto.jsonl`. Update the GitHub release to have the content of the release's changelog. -* [ ] Announce on: - * [ ] Twitter - * [ ] Discord - * [ ] OpenCollective * [ ] Update Tidelift metadata -* [ ] If this was a 1.26.x release, add changelog to the `main` branch diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d0a22bc20..97c7f1602e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: - macos-latest - windows-latest - ubuntu-20.04 # OpenSSL 1.1.1 - - ubuntu-22.04 # OpenSSL 3.0 + - ubuntu-latest # OpenSSL 3.0 nox-session: [''] include: - experimental: false @@ -89,7 +89,7 @@ jobs: os: ubuntu-22.04 runs-on: ${{ matrix.os }} - name: ${{ fromJson('{"macos-latest":"macOS","windows-latest":"Windows","ubuntu-latest":"Ubuntu","ubuntu-20.04":"Ubuntu 20.04 (OpenSSL 1.1.1)","ubuntu-22.04":"Ubuntu 22.04 (OpenSSL 3.0)"}')[matrix.os] }} ${{ matrix.python-version }} ${{ matrix.nox-session }} + name: ${{ fromJson('{"macos-latest":"macOS","windows-latest":"Windows","ubuntu-latest":"Ubuntu","ubuntu-20.04":"Ubuntu 20.04 (OpenSSL 1.1.1)","ubuntu-latest":"Ubuntu Latest (OpenSSL 3+)"}')[matrix.os] }} ${{ matrix.python-version }} ${{ matrix.nox-session }} continue-on-error: ${{ matrix.experimental }} timeout-minutes: 40 steps: @@ -208,7 +208,7 @@ jobs: - --log.level=INFO httpbin: - image: mccutchen/go-httpbin:v2.10.0 + image: mccutchen/go-httpbin:v2.11.1 restart: unless-stopped depends_on: proxy: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 520b1f2a22..7391d28c84 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -14,7 +14,7 @@ jobs: fail-fast: false matrix: downstream: [botocore, niquests] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 30 steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dc05ec6820..98e8082820 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: - id: isort - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 additional_dependencies: [flake8-2020] diff --git a/CHANGES.rst b/CHANGES.rst index cd05205056..1b465b8ad3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,67 @@ +2.2.900 (2023-11-01) +==================== + +- Added support for in-memory client (intermediary) certificate to be used with mTLS. + This feature compensate for the complete removal of ``pyOpenSSL``. Unfortunately it is only + available on Linux, OpenBSD, and FreeBSD. Using newly added ``cert_data`` and ``key_data`` arguments + in ``HTTPSConnection`` and ``HTTPSPoolConnection`` you will be capable of passing the certificate along with + its key without getting nowhere near your filesystem. + MacOS and Windows are not concerned by this feature when using HTTP/1.1, and HTTP/2 with TLS over TCP. +- Removed remnant ``SSLTransport.makefile`` as it was built to circumvent a legacy constraint when urllib3 depended upon + ``http.client``. +- Bumped minimum requirement for ``qh3`` to version 0.13.0 in order to support in-memory client certificate (mTLS). +- Symbolic complete detachment from ``http.client``. Removed all references and imports to ``http.client``. Farewell! +- Changed the default ciphers in default SSLContext for an **increased** security level. + *Rational:* Earlier in v2.1.901 we initialized the SSLContext ciphers with the value ``DEFAULT`` but after much + consideration, after we saw that the associated ciphers (e.g. ``DEFAULT`` from OpenSSL) includes some weak suites + we decided to inject a rather safer and limited cipher suite. It is based on https://ssl-config.mozilla.org + Starting now, urllib3.future will match Mozilla cipher recommendations (intermediary) and will regularly update the suite. +- Added support for multiplexed connection. HTTP/2 and HTTP/3 can benefit from this. + urllib3.future no longer blocks when ``urlopen(...)`` is invoked using ``multiplexed=True``, and return + a ``ResponsePromise`` instead of a ``HTTPResponse``. You may dispatch as much requests as the protocol + permits you (concurrent stream) and then retrieve the response(s) using the ``get_response(...)``. + ``get_response(...)`` can take up to one kwarg to specify the target promise, if none specified, will retrieve + the first available response. ``multiplexed`` is set to False by default and will likely be the default for a long + time. + Here is an example:: + + from urllib3 import PoolManager + + with PoolManager() as pm: + promise0 = pm.urlopen("GET", "https://pie.dev/delay/3", multiplexed=True) + # + promise1 = pm.urlopen("GET", "https://pie.dev/delay/1", multiplexed=True) + # + response0 = pm.get_response() + # the second request arrived first + response0.json()["url"] # https://pie.dev/delay/1 + # the first arrived last + response1 = pm.get_response() + response1.json()["url"] # https://pie.dev/delay/3 + + or you may do:: + + from urllib3 import PoolManager + + with PoolManager() as pm: + promise0 = pm.urlopen("GET", "https://pie.dev/delay/3", multiplexed=True) + # + promise1 = pm.urlopen("GET", "https://pie.dev/delay/1", multiplexed=True) + # + response0 = pm.get_response(promise=promise0) + # forcing retrieving promise0 + response0.json()["url"] # https://pie.dev/delay/3 + # then pick first available + response1 = pm.get_response() + response1.json()["url"] # https://pie.dev/delay/1 + + You may do multiplexing using ``PoolManager``, and ``HTTPSPoolConnection``. Connection upgrade + to HTTP/3 cannot be done until all in-flight requests are completed. + Be aware that a non-capable connection (e.g. HTTP/1.1) will just ignore the ``multiplexed=True`` setting + and act traditionally. +- Connection are now released into their respective pool when the connection support multiplexing (HTTP/2, HTTP/3) + before the response has been consumed. This allows to have multiple response half-consumed from a single connection. + 2.1.903 (2023-10-23) ==================== diff --git a/README.md b/README.md index 4485a8ce56..4a177bf926 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,9 @@
urllib3.future is as BoringSSL is to OpenSSL but to urllib3 (except support is available!)

-urllib3 is a powerful, *user-friendly* HTTP client for Python. urllib3.future goes beyond supported features while remaining -mostly compatible. -urllib3.future brings many critical features that are missing from the Python -standard libraries: +⚡ urllib3.future is a powerful, *user-friendly* HTTP client for Python. +⚡ urllib3.future goes beyond supported features while remaining compatible. +⚡ urllib3.future brings many critical features that are missing from the Python standard libraries: - Thread safety. - Connection pooling. @@ -22,10 +21,11 @@ standard libraries: - Helpers for retrying requests and dealing with HTTP redirects. - Support for gzip, deflate, brotli, and zstd encoding. - HTTP/1.1, HTTP/2 and HTTP/3 support. +- Multiplexed connection. - Proxy support for HTTP and SOCKS. - 100% test coverage. -urllib3 is powerful and easy to use: +urllib3.future is powerful and easy to use: ```python >>> import urllib3 @@ -46,12 +46,11 @@ urllib3.future can be installed with [pip](https://pip.pypa.io): $ python -m pip install urllib3.future ``` -⚠️ Installing urllib3.future shadows the actual urllib3 package (_depending on installation order_) and you should -carefully weigh the impacts. The semver will always be like _MAJOR.MINOR.9PP_ like 2.0.941, the patch node -is always greater or equal to 900. +⚠️ Installing urllib3.future shadows the actual urllib3 package (_depending on installation order_). +The semver will always be like _MAJOR.MINOR.9PP_ like 2.0.941, the patch node is always greater or equal to 900. Support for bugs or improvements is served in this repository. We regularly sync this fork -with the main branch of urllib3/urllib3. +with the main branch of urllib3/urllib3 against bugfixes and security patches if applicable. ## Compatibility with downstream @@ -64,15 +63,7 @@ python -m pip install requests python -m pip install urllib3.future ``` -| Package | Is compatible? | Notes | -|------------------|----------------|-------------------------------------------------------------------------------------------------------------------------------------------------| -| requests | ✅ | Invalid chunked transmission may raises ConnectionError instead of ChunkedEncodingError. Use of Session() is required to enable HTTP/3 support. | -| HTTPie | ✅ | Require plugin `httpie-next` to be installed or wont be able to upgrade to HTTP/3 (QUIC/Alt-Svc Cache Layer) | -| pip | 🛑 | Cannot use the fork because of vendored urllib3 v1.x | -| openapigenerator | ✅ | Simply patch generated `setup.py` requirement and replace urllib3 to urllib3.future | - -Want to report an incompatibility? Open an issue in that repository. -All projects that depends on listed *compatible* package should work as-is. +We suggest using the package **Niquests** as replacement for **Requests**. It leverage urllib3.future capabilities. ## Documentation diff --git a/dev-requirements.txt b/dev-requirements.txt index 3a1aae249f..0d1cc8b02c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -10,4 +10,3 @@ cryptography==39.0.2;implementation_name=="pypy" and implementation_version<"7.3 cryptography==41.0.2;implementation_name!="pypy" or implementation_version>="7.3.10" backports.zoneinfo==0.2.1;python_version<"3.9" towncrier==21.9.0 -pytest-memray==1.4.0;python_version>="3.8" and python_version<"3.12" and sys_platform!="win32" and implementation_name=="cpython" diff --git a/docs/advanced-usage.rst b/docs/advanced-usage.rst index a427197bd2..107b4ef0e3 100644 --- a/docs/advanced-usage.rst +++ b/docs/advanced-usage.rst @@ -688,3 +688,99 @@ It takes a ``set`` of ``HttpVersion`` like so: HTTP/3 require installing ``qh3`` package if not automatically available. Setting disabled_svn has no effect otherwise. Also, you cannot disable HTTP/1.1 at the current state of affairs. + +Multiplexed connection +---------------------- + +Since the version 2.2 you can emit multiple concurrent requests and retrieve the responses later. +A new keyword argument is available in ``PoolManager``, ``HTTPPoolConnection`` through the following methods: + +- :meth:`~urllib3.PoolManager.request` +- :meth:`~urllib3.PoolManager.urlopen` +- :meth:`~urllib3.PoolManager.request_encode_url` +- :meth:`~urllib3.PoolManager.request_encode_body` + +When you omit ``multiplexed=...`` it default to the old behavior of waiting upon the response and return a :class:`~urllib3.response.HTTPResponse` +otherwise if you specify ``multiplexed=True`` it will return a :class:`~urllib3.backend.ResponsePromise` instead. + +Here is an example:: + + from urllib3 import PoolManager + + with PoolManager() as pm: + promise0 = pm.urlopen("GET", "https://pie.dev/delay/3", multiplexed=True) + # + promise1 = pm.urlopen("GET", "https://pie.dev/delay/1", multiplexed=True) + # + response0 = pm.get_response() + # the second request arrived first + response0.json()["url"] # https://pie.dev/delay/1 + # the first arrived last + response1 = pm.get_response() + response1.json()["url"] # https://pie.dev/delay/3 + + or you may do:: + + from urllib3 import PoolManager + + with PoolManager() as pm: + promise0 = pm.urlopen("GET", "https://pie.dev/delay/3", multiplexed=True) + # + promise1 = pm.urlopen("GET", "https://pie.dev/delay/1", multiplexed=True) + # + response0 = pm.get_response(promise=promise0) + # forcing retrieving promise0 + response0.json()["url"] # https://pie.dev/delay/3 + # then pick first available + response1 = pm.get_response() + response1.json()["url"] # https://pie.dev/delay/1 + +.. note:: You cannot expect the connection upgrade to HTTP/3 if all in-flight request aren't consumed. + +.. warning:: Using ``multiplexed=True`` if the target connection does not support it is ignored and assume you meant ``multiplexed=False``. It will raise a warning in a future version. + +Associate a promise to its response +----------------------------------- + +When issuing concurrent request using ``multiplexed=True`` and want to retrieve +the responses in whatever order they may come, you may want to clearly identify the originating promise. + +To identify with certainty:: + + from urllib3 import PoolManager + + with PoolManager() as pm: + promise0 = pm.urlopen("GET", "https://pie.dev/delay/3", multiplexed=True) + # + promise1 = pm.urlopen("GET", "https://pie.dev/delay/1", multiplexed=True) + # + response = pm.get_response() + # verify that response is linked to second promise + response.is_from_promise(promise0) + # True! + response.is_from_promise(promise1) + # False. + +In-memory client (mTLS) certificate +----------------------------------- + +.. note:: Available since version 2.2 + +Using newly added ``cert_data`` and ``key_data`` arguments in ``HTTPSConnection``, ``HTTPSPoolConnection`` and ``PoolManager``. +you will be capable of passing the certificate along with its key without getting nowhere near your filesystem. + +.. warning:: When connected to a TLS over TCP, this is only supported with Linux, FreeBSD, and OpenBSD. When connected over QUIC (e.g. HTTP/3) it is broadly supported. + +This feature compensate for the complete removal of ``pyOpenSSL``. + +You may give your certificate to urllib3.future this way:: + + with HTTPSConnectionPool( + self.host, + self.port, + key_data=CLIENT_INTERMEDIATE_KEY, + cert_data=CLIENT_INTERMEDIATE_PEM, + ) as https_pool: + r = https_pool.request("GET", "/") + +.. note:: If your platform isn't served by this feature it will raise a warning and ignore the certificate. diff --git a/docs/conf.py b/docs/conf.py index ad6abd66a1..d780eda9bc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -107,8 +107,14 @@ ("py:class", "_TYPE_TIMEOUT"), ("py:class", "_TYPE_FIELD_VALUE"), ("py:class", "_TYPE_BODY"), + ("py:class", "_TYPE_TIMEOUT_INTERNAL"), + ("py:class", "_TYPE_FAILEDTELL"), + ("py:class", "_TYPE_FIELD_VALUE_TUPLE"), + ("py:class", "_TYPE_FIELDS"), + ("py:class", "_TYPE_ENCODE_URL_FIELDS"), ("py:class", "_HttplibHTTPResponse"), ("py:class", "_HttplibHTTPMessage"), + ("py:class", "Message"), ("py:class", "TracebackType"), ("py:class", "Literal"), ("py:class", "email.errors.MessageDefect"), @@ -118,12 +124,15 @@ ("py:class", "SSLTransportType"), ("py:class", "VerifyMode"), ("py:class", "_ssl._SSLContext"), + ("py:func", "ssl.wrap_socket"), ("py:class", "urllib3._collections.HTTPHeaderDict"), ("py:class", "urllib3._collections.RecentlyUsedContainer"), ("py:class", "urllib3._request_methods.RequestMethods"), ("py:class", "urllib3.contrib.socks._TYPE_SOCKS_OPTIONS"), ("py:class", "urllib3.util.timeout._TYPE_DEFAULT"), - ("py:class", "BaseHTTPConnection"), + ("py:class", "urllib3.util.request._TYPE_FAILEDTELL"), + ("py:class", "BaseHTTPResponse"), + ("py:class", "urllib3.response.BaseHTTPResponse"), ("py:class", "connection._TYPE_SOCKET_OPTIONS"), ("py:class", "urllib3.backend.httplib._PatchedHTTPConnection"), ("py:class", "urllib3.backend._base.LowLevelResponse"), diff --git a/docs/index.rst b/docs/index.rst index 1cf90120d6..be5c134975 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,18 +12,19 @@ urllib3 contributing changelog -urllib3 is a powerful, *user-friendly* HTTP client for Python. +⚡ urllib3.future is a powerful, *user-friendly* HTTP client for Python. +⚡ urllib3.future goes beyond supported features while remaining compatible. -urllib3 brings many critical features that are missing from the Python -standard libraries: +⚡ urllib3.future brings many critical features that are missing from the Python standard libraries: - Thread safety. - Connection pooling. -- Client-side TLS/SSL verification. +- Client-side SSL/TLS verification. - File uploads with multipart encoding. - Helpers for retrying requests and dealing with HTTP redirects. - Support for gzip, deflate, brotli, and zstd encoding. - HTTP/1.1, HTTP/2 and HTTP/3 support. +- Multiplexed connection. - Proxy support for HTTP and SOCKS. - 100% test coverage. @@ -58,7 +59,7 @@ The :doc:`reference/index` documentation provides API-level documentation. License ------- -urllib3 is made available under the MIT License. For more details, see `LICENSE.txt `_. +urllib3.future is made available under the MIT License. For more details, see `LICENSE.txt `_. Contributing ------------ diff --git a/docs/reference/urllib3.backend.rst b/docs/reference/urllib3.backend.rst index f32e50c8f1..dbe9b88b6d 100644 --- a/docs/reference/urllib3.backend.rst +++ b/docs/reference/urllib3.backend.rst @@ -14,4 +14,10 @@ Backends .. autoclass:: urllib3.backend.HttpVersion :members: +.. autoclass:: urllib3.backend.ResponsePromise + :members: + .. autoclass:: urllib3.backend.QuicPreemptiveCacheType + +.. autoclass:: urllib3.backend.ConnectionInfo + :members: diff --git a/docs/reference/urllib3.connection.rst b/docs/reference/urllib3.connection.rst index 8bd7b258bd..147ab95411 100644 --- a/docs/reference/urllib3.connection.rst +++ b/docs/reference/urllib3.connection.rst @@ -12,6 +12,6 @@ Connections :members: :show-inheritance: -.. autoclass:: urllib3.connection.ProxyConfig +.. autoclass:: urllib3._typing.ProxyConfig :members: :show-inheritance: diff --git a/docs/v2-migration-guide.rst b/docs/v2-migration-guide.rst index 0ad1381be0..70e463b910 100644 --- a/docs/v2-migration-guide.rst +++ b/docs/v2-migration-guide.rst @@ -14,27 +14,7 @@ So unless you're in a specific situation you should notice no changes! 🎉 .. note:: If you have difficulty migrating to v2.0 or following this guide - you can `open an issue on GitHub `_ - or reach out in `our community Discord channel `_. - - -Timeline for deprecations and breaking changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The 2.x initial release schedule will look like this: - -* urllib3 ``v2.0.0-alpha1`` will be released in November 2022. This release - contains **minor breaking changes and deprecation warnings for other breaking changes**. - There may be other pre-releases to address fixes before v2.0.0 is released. -* urllib3 ``v2.0.0`` will be released in early 2023 after some initial integration testing - against dependent packages and fixing of bug reports. -* urllib3 ``v2.1.0`` will be released in the summer of 2023 with **all breaking changes - being warned about in v2.0.0**. - -.. warning:: - - Please take the ``DeprecationWarnings`` you receive when migrating from v1.x to v2.0 seriously - as they will become errors after 2.1.0 is released. + you can `open an issue on GitHub `_. What are the important changes? @@ -46,16 +26,17 @@ Here's a short summary of which changes in urllib3 v2.0 are most important: - Removed support for OpenSSL versions older than 1.1.1. - Removed support for Python implementations that aren't CPython or PyPy3 (previously supported Google App Engine, Jython). - Removed the ``urllib3.contrib.ntlmpool`` module. -- Deprecated the ``urllib3.contrib.pyopenssl``, ``urllib3.contrib.securetransport`` modules, will be removed in v2.1.0. -- Deprecated the ``urllib3[secure]`` extra, will be removed in v2.1.0. +- Deprecated the ``urllib3.contrib.pyopenssl`` module, made inoperant in v2.1.0. +- Deprecated the ``urllib3.contrib.securetransport`` module, will be removed in v2.1.0. +- Deprecated the ``urllib3[secure]`` extra, made inoperant in v2.1.0. - Deprecated the ``HTTPResponse.getheaders()`` method in favor of ``HTTPResponse.headers``, will be removed in v2.1.0. - Deprecated the ``HTTPResponse.getheader(name, default)`` method in favor of ``HTTPResponse.headers.get(name, default)``, will be removed in v2.1.0. - Deprecated URLs without a scheme (ie 'https://') and will be raising an error in a future version of urllib3. - Changed the default minimum TLS version to TLS 1.2 (previously was TLS 1.0). - Removed support for verifying certificate hostnames via ``commonName``, now only ``subjectAltName`` is used. -- Removed the default set of TLS ciphers, instead now urllib3 uses the list of ciphers configured by the system. +- Removed the default set of TLS ciphers, instead now urllib3 uses the list of ciphers recommended by Mozilla. -For a full list of changes you can look at `the changelog `_. +For a full list of changes you can look at `the changelog `_. Migrating as a package maintainer? @@ -76,20 +57,20 @@ ensure your package allows for both urllib3 1.26.x and 2.0 to be used: # setup.py (setuptools) setup( ... - install_requires=["urllib3>=1.26,<3"] + install_requires=["urllib3.future>=2,<3"] ) # pyproject.toml (hatch) [project] dependencies = [ - "urllib3>=1.26,<3" + "urllib3.future>=2,<3" ] -Next you should try installing urllib3 v2.0 locally and run your test suite. +Next you should try installing urllib3.future locally and run your test suite. .. code-block:: bash - $ python -m pip install -U --pre 'urllib3>=2.0.0a1' + $ python -m pip install -U urllib3.future Because there are many ``DeprecationWarnings`` you should ensure that you're @@ -130,9 +111,8 @@ that supports both urllib3 v1.26.x and v2.x. .. note:: - If you're not able to support both 1.26.x and v2.0 of urllib3 at the same time with your package please - `open an issue on GitHub `_ or reach out in - `our community Discord channel `_. + If you're not able to support urllib3.future with your package please + `open an issue on GitHub `_. Migrating as an application developer? @@ -331,7 +311,7 @@ from auto-complete. No more confusion with ``**kwargs``! If you're one of the rare few who is subclassing connections or responses you should take a closer look at detailed changes - in `the changelog `_. + in `the changelog `_. **🔐 Modern security by default** diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index 0b6d1a09be..e1d07f6504 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -325,7 +325,7 @@ def part( return part with monkeypatch.context() as m: - m.setattr(HTTPConnection, "request", call_and_mark(orig_request)) + m.setattr(HTTPConnection, "request", call_and_mark(orig_request)) # type: ignore[arg-type] yield @classmethod diff --git a/noxfile.py b/noxfile.py index 2dcd64991e..16da17ed82 100644 --- a/noxfile.py +++ b/noxfile.py @@ -2,7 +2,6 @@ import os import shutil -import sys import nox @@ -22,17 +21,6 @@ def tests_impl( session.run("python", "--version") session.run("python", "-c", "import struct; print(struct.calcsize('P') * 8)") - memray_supported = True - if ( - sys.implementation.name != "cpython" - or sys.version_info < (3, 8) - or sys.version_info.releaselevel != "final" - or sys.version_info >= (3, 12) - ): - memray_supported = False # pytest-memray requires CPython 3.8+ - elif sys.platform == "win32": - memray_supported = False - # Inspired from https://hynek.me/articles/ditch-codecov-python/ # We use parallel mode and then combine in a later CI step session.run( @@ -44,7 +32,6 @@ def tests_impl( "--parallel-mode", "-m", "pytest", - *("--memray", "--hide-memray-summary") if memray_supported else (), "-v", "-ra", f"--color={'yes' if 'GITHUB_ACTIONS' in os.environ else 'auto'}", diff --git a/pyproject.toml b/pyproject.toml index 6074959089..1a96d46a32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,9 +6,9 @@ build-backend = "hatchling.build" [project] name = "urllib3-future" -description = "Fork of mainstream urllib3 but ahead of his time" +description = "urllib3.future is a powerful HTTP 1.1, 2, and 3 client" readme = "README.md" -keywords = ["urllib", "httplib", "threadsafe", "filepost", "http", "https", "ssl", "pooling"] +keywords = ["urllib", "httplib", "threadsafe", "filepost", "http", "https", "ssl", "pooling", "multiplexed", "concurrent"] authors = [ {name = "Andrey Petrov", email = "andrey.petrov@shazow.net"} ] @@ -37,7 +37,7 @@ classifiers = [ requires-python = ">=3.7" dynamic = ["version"] dependencies = [ - "qh3>=0.11.3,<1.0.0; (platform_system == 'Darwin' or platform_system == 'Windows' or platform_system == 'Linux') and (platform_python_implementation == 'CPython' or (platform_python_implementation == 'PyPy' and python_version >= '3.8' and python_version < '3.11'))", + "qh3>=0.13.0,<1.0.0; (platform_system == 'Darwin' or platform_system == 'Windows' or platform_system == 'Linux') and (platform_python_implementation == 'CPython' or (platform_python_implementation == 'PyPy' and python_version >= '3.8' and python_version < '3.11'))", "h11>=0.11.0,<1.0.0", "h2>=4.0.0,<5.0.0", ] @@ -55,7 +55,7 @@ socks = [ "PySocks>=1.5.6,<2.0,!=1.5.7", ] qh3 = [ - "qh3>=0.11.2,<1.0.0", + "qh3>=0.13.0,<1.0.0", ] [project.urls] diff --git a/setup.cfg b/setup.cfg index 5dd55b4b71..759b05951b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,4 @@ [flake8] -ignore = E501, E203, W503, W504 +ignore = E501, E203, W503, W504, E721 exclude=./docs/conf.py max-line-length=99 diff --git a/src/urllib3/__init__.py b/src/urllib3/__init__.py index 9e434cade3..85a0322154 100644 --- a/src/urllib3/__init__.py +++ b/src/urllib3/__init__.py @@ -15,7 +15,7 @@ from ._collections import HTTPHeaderDict from ._typing import _TYPE_BODY, _TYPE_FIELDS from ._version import __version__ -from .backend import ConnectionInfo, HttpVersion +from .backend import ConnectionInfo, HttpVersion, ResponsePromise from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url from .filepost import encode_multipart_formdata from .poolmanager import PoolManager, ProxyManager, proxy_from_url @@ -67,6 +67,7 @@ "BaseHTTPResponse", "HttpVersion", "ConnectionInfo", + "ResponsePromise", ) logging.getLogger(__name__).addHandler(NullHandler()) @@ -135,14 +136,23 @@ def request( retries: Retry | bool | int | None = None, timeout: Timeout | float | int | None = 3, json: typing.Any | None = None, -) -> BaseHTTPResponse: + multiplexed: bool = False, +) -> HTTPResponse: """ A convenience, top-level request method. It uses a module-global ``PoolManager`` instance. Therefore, its side effects could be shared across dependencies relying on it. To avoid side effects create a new ``PoolManager`` instance and use it instead. - The method does not accept low-level ``**urlopen_kw`` keyword arguments. + The method does not accept low-level ``**urlopen_kw`` keyword arguments neither does + it issue multiplexed/concurrent request. """ + if multiplexed: + warnings.warn( + "Setting multiplexed=True in urllib3.request top-level function is a no-op. " + "Use a local PoolManager or HTTPPoolConnection instead.", + UserWarning, + ) + return _DEFAULT_POOL.request( method, url, diff --git a/src/urllib3/_constant.py b/src/urllib3/_constant.py new file mode 100644 index 0000000000..998b80ea4c --- /dev/null +++ b/src/urllib3/_constant.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +import typing +from enum import IntEnum + + +class HTTPStatus(IntEnum): + """HTTP status codes and reason phrases + + Status codes from the following RFCs are all observed: + + * RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616 + * RFC 6585: Additional HTTP Status Codes + * RFC 3229: Delta encoding in HTTP + * RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518 + * RFC 5842: Binding Extensions to WebDAV + * RFC 7238: Permanent Redirect + * RFC 2295: Transparent Content Negotiation in HTTP + * RFC 2774: An HTTP Extension Framework + * RFC 7725: An HTTP Status Code to Report Legal Obstacles + * RFC 7540: Hypertext Transfer Protocol Version 2 (HTTP/2) + * RFC 2324: Hyper Text Coffee Pot Control Protocol (HTCPCP/1.0) + * RFC 8297: An HTTP Status Code for Indicating Hints + * RFC 8470: Using Early Data in HTTP + """ + + phrase: str + description: str + standard: bool + + def __new__( + cls, value: int, phrase: str, description: str = "", is_standard: bool = True + ) -> HTTPStatus: + obj = int.__new__(cls, value) + obj._value_ = value + + obj.phrase = phrase + obj.description = description + obj.standard = is_standard + return obj + + # informational + CONTINUE = 100, "Continue", "Request received, please continue" + SWITCHING_PROTOCOLS = ( + 101, + "Switching Protocols", + "Switching to new protocol; obey Upgrade header", + ) + PROCESSING = 102, "Processing" + EARLY_HINTS = 103, "Early Hints" + + # success + OK = 200, "OK", "Request fulfilled, document follows" + CREATED = 201, "Created", "Document created, URL follows" + ACCEPTED = (202, "Accepted", "Request accepted, processing continues off-line") + NON_AUTHORITATIVE_INFORMATION = ( + 203, + "Non-Authoritative Information", + "Request fulfilled from cache", + ) + NO_CONTENT = 204, "No Content", "Request fulfilled, nothing follows" + RESET_CONTENT = 205, "Reset Content", "Clear input form for further input" + PARTIAL_CONTENT = 206, "Partial Content", "Partial content follows" + MULTI_STATUS = 207, "Multi-Status" + ALREADY_REPORTED = 208, "Already Reported" + IM_USED = 226, "IM Used" + + # redirection + MULTIPLE_CHOICES = ( + 300, + "Multiple Choices", + "Object has several resources -- see URI list", + ) + MOVED_PERMANENTLY = ( + 301, + "Moved Permanently", + "Object moved permanently -- see URI list", + ) + FOUND = 302, "Found", "Object moved temporarily -- see URI list" + SEE_OTHER = 303, "See Other", "Object moved -- see Method and URL list" + NOT_MODIFIED = (304, "Not Modified", "Document has not changed since given time") + USE_PROXY = ( + 305, + "Use Proxy", + "You must use proxy specified in Location to access this resource", + ) + TEMPORARY_REDIRECT = ( + 307, + "Temporary Redirect", + "Object moved temporarily -- see URI list", + ) + PERMANENT_REDIRECT = ( + 308, + "Permanent Redirect", + "Object moved permanently -- see URI list", + ) + + # client error + BAD_REQUEST = (400, "Bad Request", "Bad request syntax or unsupported method") + UNAUTHORIZED = (401, "Unauthorized", "No permission -- see authorization schemes") + PAYMENT_REQUIRED = (402, "Payment Required", "No payment -- see charging schemes") + FORBIDDEN = (403, "Forbidden", "Request forbidden -- authorization will not help") + NOT_FOUND = (404, "Not Found", "Nothing matches the given URI") + METHOD_NOT_ALLOWED = ( + 405, + "Method Not Allowed", + "Specified method is invalid for this resource", + ) + NOT_ACCEPTABLE = (406, "Not Acceptable", "URI not available in preferred format") + PROXY_AUTHENTICATION_REQUIRED = ( + 407, + "Proxy Authentication Required", + "You must authenticate with this proxy before proceeding", + ) + REQUEST_TIMEOUT = (408, "Request Timeout", "Request timed out; try again later") + CONFLICT = 409, "Conflict", "Request conflict" + GONE = (410, "Gone", "URI no longer exists and has been permanently removed") + LENGTH_REQUIRED = (411, "Length Required", "Client must specify Content-Length") + PRECONDITION_FAILED = ( + 412, + "Precondition Failed", + "Precondition in headers is false", + ) + REQUEST_ENTITY_TOO_LARGE = (413, "Request Entity Too Large", "Entity is too large") + REQUEST_URI_TOO_LONG = (414, "Request-URI Too Long", "URI is too long") + UNSUPPORTED_MEDIA_TYPE = ( + 415, + "Unsupported Media Type", + "Entity body in unsupported format", + ) + REQUESTED_RANGE_NOT_SATISFIABLE = ( + 416, + "Requested Range Not Satisfiable", + "Cannot satisfy request range", + ) + EXPECTATION_FAILED = ( + 417, + "Expectation Failed", + "Expect condition could not be satisfied", + ) + IM_A_TEAPOT = ( + 418, + "I'm a Teapot", + "Server refuses to brew coffee because it is a teapot.", + ) + MISDIRECTED_REQUEST = ( + 421, + "Misdirected Request", + "Server is not able to produce a response", + ) + UNPROCESSABLE_ENTITY = 422, "Unprocessable Entity" + LOCKED = 423, "Locked" + FAILED_DEPENDENCY = 424, "Failed Dependency" + TOO_EARLY = 425, "Too Early" + UPGRADE_REQUIRED = 426, "Upgrade Required" + PRECONDITION_REQUIRED = ( + 428, + "Precondition Required", + "The origin server requires the request to be conditional", + ) + TOO_MANY_REQUESTS = ( + 429, + "Too Many Requests", + "The user has sent too many requests in " + 'a given amount of time ("rate limiting")', + ) + REQUEST_HEADER_FIELDS_TOO_LARGE = ( + 431, + "Request Header Fields Too Large", + "The server is unwilling to process the request because its header " + "fields are too large", + ) + UNAVAILABLE_FOR_LEGAL_REASONS = ( + 451, + "Unavailable For Legal Reasons", + "The server is denying access to the " + "resource as a consequence of a legal demand", + ) + + # server errors + INTERNAL_SERVER_ERROR = ( + 500, + "Internal Server Error", + "Server got itself in trouble", + ) + NOT_IMPLEMENTED = (501, "Not Implemented", "Server does not support this operation") + BAD_GATEWAY = (502, "Bad Gateway", "Invalid responses from another server/proxy") + SERVICE_UNAVAILABLE = ( + 503, + "Service Unavailable", + "The server cannot process the request due to a high load", + ) + GATEWAY_TIMEOUT = ( + 504, + "Gateway Timeout", + "The gateway server did not receive a timely response", + ) + HTTP_VERSION_NOT_SUPPORTED = ( + 505, + "HTTP Version Not Supported", + "Cannot fulfill request", + ) + VARIANT_ALSO_NEGOTIATES = 506, "Variant Also Negotiates" + INSUFFICIENT_STORAGE = 507, "Insufficient Storage" + LOOP_DETECTED = 508, "Loop Detected" + NOT_EXTENDED = 510, "Not Extended" + NETWORK_AUTHENTICATION_REQUIRED = ( + 511, + "Network Authentication Required", + "The client needs to authenticate to gain network access", + ) + + +# another hack to maintain backwards compatibility +# Mapping status codes to official W3C names +responses: typing.Mapping[int, str] = { + v: v.phrase for v in HTTPStatus.__members__.values() +} + +# Default value for `blocksize` - a new parameter introduced to +# http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7 +DEFAULT_BLOCKSIZE: int = 16384 + +# Mozilla TLS recommendations for ciphers +# General-purpose servers with a variety of clients, recommended for almost all systems. +MOZ_INTERMEDIATE_CIPHERS: str = "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305" diff --git a/src/urllib3/_request_methods.py b/src/urllib3/_request_methods.py index 600bdcb52c..923027c979 100644 --- a/src/urllib3/_request_methods.py +++ b/src/urllib3/_request_methods.py @@ -7,7 +7,12 @@ from ._collections import HTTPHeaderDict from ._typing import _TYPE_BODY, _TYPE_ENCODE_URL_FIELDS, _TYPE_FIELDS from .filepost import encode_multipart_formdata -from .response import BaseHTTPResponse +from .response import HTTPResponse + +if typing.TYPE_CHECKING: + from typing_extensions import Literal + + from .backend import ResponsePromise __all__ = ["RequestMethods"] @@ -46,6 +51,36 @@ class RequestMethods: def __init__(self, headers: typing.Mapping[str, str] | None = None) -> None: self.headers = headers or {} + @typing.overload + def urlopen( + self, + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + encode_multipart: bool = True, + multipart_boundary: str | None = None, + *, + multiplexed: Literal[False] = ..., + **kw: typing.Any, + ) -> HTTPResponse: + ... + + @typing.overload + def urlopen( + self, + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + encode_multipart: bool = True, + multipart_boundary: str | None = None, + *, + multiplexed: Literal[True], + **kw: typing.Any, + ) -> ResponsePromise: + ... + def urlopen( self, method: str, @@ -55,12 +90,42 @@ def urlopen( encode_multipart: bool = True, multipart_boundary: str | None = None, **kw: typing.Any, - ) -> BaseHTTPResponse: # Abstract + ) -> HTTPResponse | ResponsePromise: raise NotImplementedError( "Classes extending RequestMethods must implement " "their own ``urlopen`` method." ) + @typing.overload + def request( + self, + method: str, + url: str, + body: _TYPE_BODY | None = ..., + fields: _TYPE_FIELDS | None = ..., + headers: typing.Mapping[str, str] | None = ..., + json: typing.Any | None = ..., + *, + multiplexed: Literal[False] = ..., + **urlopen_kw: typing.Any, + ) -> HTTPResponse: + ... + + @typing.overload + def request( + self, + method: str, + url: str, + body: _TYPE_BODY | None = ..., + fields: _TYPE_FIELDS | None = ..., + headers: typing.Mapping[str, str] | None = ..., + json: typing.Any | None = ..., + *, + multiplexed: Literal[True], + **urlopen_kw: typing.Any, + ) -> ResponsePromise: + ... + def request( self, method: str, @@ -70,7 +135,7 @@ def request( headers: typing.Mapping[str, str] | None = None, json: typing.Any | None = None, **urlopen_kw: typing.Any, - ) -> BaseHTTPResponse: + ) -> HTTPResponse | ResponsePromise: """ Make a request using :meth:`urlopen` with the appropriate encoding of ``fields`` based on the ``method`` used. @@ -110,18 +175,48 @@ def request( **urlopen_kw, ) else: - return self.request_encode_body( - method, url, fields=fields, headers=headers, **urlopen_kw + return self.request_encode_body( # type: ignore[no-any-return] + method, + url, + fields=fields, + headers=headers, + **urlopen_kw, ) + @typing.overload + def request_encode_url( + self, + method: str, + url: str, + fields: _TYPE_ENCODE_URL_FIELDS | None = ..., + headers: typing.Mapping[str, str] | None = ..., + *, + multiplexed: Literal[False] = ..., + **urlopen_kw: typing.Any, + ) -> HTTPResponse: + ... + + @typing.overload + def request_encode_url( + self, + method: str, + url: str, + fields: _TYPE_ENCODE_URL_FIELDS | None = ..., + headers: typing.Mapping[str, str] | None = ..., + *, + multiplexed: Literal[True], + **urlopen_kw: typing.Any, + ) -> ResponsePromise: + ... + def request_encode_url( self, method: str, url: str, fields: _TYPE_ENCODE_URL_FIELDS | None = None, headers: typing.Mapping[str, str] | None = None, - **urlopen_kw: str, - ) -> BaseHTTPResponse: + **urlopen_kw: typing.Any, + ) -> HTTPResponse | ResponsePromise: """ Make a request using :meth:`urlopen` with the ``fields`` encoded in the url. This is useful for request methods like GET, HEAD, DELETE, etc. @@ -135,7 +230,37 @@ def request_encode_url( if fields: url += "?" + urlencode(fields) - return self.urlopen(method, url, **extra_kw) + return self.urlopen(method, url, **extra_kw) # type: ignore[no-any-return] + + @typing.overload + def request_encode_body( + self, + method: str, + url: str, + fields: _TYPE_FIELDS | None = ..., + headers: typing.Mapping[str, str] | None = ..., + encode_multipart: bool = ..., + multipart_boundary: str | None = ..., + *, + multiplexed: Literal[False] = ..., + **urlopen_kw: typing.Any, + ) -> HTTPResponse: + ... + + @typing.overload + def request_encode_body( + self, + method: str, + url: str, + fields: _TYPE_FIELDS | None = ..., + headers: typing.Mapping[str, str] | None = ..., + encode_multipart: bool = ..., + multipart_boundary: str | None = ..., + *, + multiplexed: Literal[True], + **urlopen_kw: typing.Any, + ) -> ResponsePromise: + ... def request_encode_body( self, @@ -145,8 +270,8 @@ def request_encode_body( headers: typing.Mapping[str, str] | None = None, encode_multipart: bool = True, multipart_boundary: str | None = None, - **urlopen_kw: str, - ) -> BaseHTTPResponse: + **urlopen_kw: typing.Any, + ) -> HTTPResponse | ResponsePromise: """ Make a request using :meth:`urlopen` with the ``fields`` encoded in the body. This is useful for request methods like POST, PUT, PATCH, etc. @@ -209,4 +334,4 @@ def request_encode_body( extra_kw.update(urlopen_kw) - return self.urlopen(method, url, **extra_kw) + return self.urlopen(method, url, **extra_kw) # type: ignore[no-any-return] diff --git a/src/urllib3/_version.py b/src/urllib3/_version.py index bd992feab3..cb0fbfb906 100644 --- a/src/urllib3/_version.py +++ b/src/urllib3/_version.py @@ -1,4 +1,4 @@ # This file is protected via CODEOWNERS from __future__ import annotations -__version__ = "2.1.903" +__version__ = "2.2.900" diff --git a/src/urllib3/backend/__init__.py b/src/urllib3/backend/__init__.py index 5223098d89..5a6a8adfe8 100644 --- a/src/urllib3/backend/__init__.py +++ b/src/urllib3/backend/__init__.py @@ -6,6 +6,7 @@ HttpVersion, LowLevelResponse, QuicPreemptiveCacheType, + ResponsePromise, ) from .hface import HfaceBackend @@ -16,4 +17,5 @@ "QuicPreemptiveCacheType", "LowLevelResponse", "ConnectionInfo", + "ResponsePromise", ) diff --git a/src/urllib3/backend/_base.py b/src/urllib3/backend/_base.py index a30e550726..3dabb92616 100644 --- a/src/urllib3/backend/_base.py +++ b/src/urllib3/backend/_base.py @@ -3,12 +3,15 @@ import enum import socket import typing +from base64 import b64encode +from secrets import token_bytes if typing.TYPE_CHECKING: from ssl import SSLSocket, SSLContext, TLSVersion from .._typing import _TYPE_SOCKET_OPTIONS from .._collections import HTTPHeaderDict +from .._constant import DEFAULT_BLOCKSIZE class HttpVersion(str, enum.Enum): @@ -64,10 +67,11 @@ def __init__( version: int, reason: str, headers: HTTPHeaderDict, - body: typing.Callable[[int | None], tuple[bytes, bool]] | None, + body: typing.Callable[[int | None, int | None], tuple[bytes, bool]] | None, *, authority: str | None = None, port: int | None = None, + stream_id: int | None = None, ): self.status = status self.version = version @@ -83,7 +87,20 @@ def __init__( self.authority = authority self.port = port + self._stream_id = stream_id + self.__buffer_excess: bytes = b"" + self.__promise: ResponsePromise | None = None + + @property + def from_promise(self) -> ResponsePromise | None: + return self.__promise + + @from_promise.setter + def from_promise(self, value: ResponsePromise) -> None: + if value.stream_id != self._stream_id: + raise ValueError + self.__promise = value @property def method(self) -> str: @@ -105,7 +122,7 @@ def read(self, __size: int | None = None) -> bytes: return b"" # Defensive: This is unreachable, this case is already covered higher in the stack. if self._eot is False: - data, self._eot = self.__internal_read_st(__size) + data, self._eot = self.__internal_read_st(__size, self._stream_id) # that's awkward, but rather no choice. the state machine # consume and render event regardless of your amt ! @@ -135,8 +152,64 @@ def close(self) -> None: self.closed = True -_HostPortType = typing.Tuple[str, int] -QuicPreemptiveCacheType = typing.MutableMapping[ +class ResponsePromise: + def __init__( + self, + conn: BaseBackend, + stream_id: int, + request_headers: list[tuple[bytes, bytes]], + **parameters: typing.Any, + ) -> None: + self._uid: str = b64encode(token_bytes(16)).decode("ascii") + self._conn: BaseBackend = conn + self._stream_id: int = stream_id + self._response: LowLevelResponse | None = None + self._request_headers = request_headers + self._parameters: typing.MutableMapping[str, typing.Any] = parameters + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ResponsePromise): + return False + return self.uid == other.uid + + def __repr__(self) -> str: + return f"" + + @property + def uid(self) -> str: + return self._uid + + @property + def request_headers(self) -> list[tuple[bytes, bytes]]: + return self._request_headers + + @property + def stream_id(self) -> int: + return self._stream_id + + @property + def is_ready(self) -> bool: + return self._response is not None + + @property + def response(self) -> LowLevelResponse: + if not self._response: + raise OSError + return self._response + + @response.setter + def response(self, value: LowLevelResponse) -> None: + self._response = value + + def set_parameter(self, key: str, value: typing.Any) -> None: + self._parameters[key] = value + + def get_parameter(self, key: str) -> typing.Any | None: + return self._parameters[key] if key in self._parameters else None + + +_HostPortType: typing.TypeAlias = typing.Tuple[str, int] +QuicPreemptiveCacheType: typing.TypeAlias = typing.MutableMapping[ _HostPortType, typing.Optional[_HostPortType] ] @@ -173,9 +246,9 @@ def __init__( port: int | None = None, timeout: int | float | None = -1, source_address: tuple[str, int] | None = None, - blocksize: int = 8192, + blocksize: int = DEFAULT_BLOCKSIZE, *, - socket_options: None | _TYPE_SOCKET_OPTIONS = default_socket_options, + socket_options: _TYPE_SOCKET_OPTIONS | None = default_socket_options, disabled_svn: set[HttpVersion] | None = None, preemptive_quic_cache: QuicPreemptiveCacheType | None = None, ): @@ -197,18 +270,23 @@ def __init__( self._tunnel_scheme: str | None = None self._tunnel_headers: typing.Mapping[str, str] = dict() - self._disabled_svn = disabled_svn or set() + self._disabled_svn = disabled_svn if disabled_svn is not None else set() self._preemptive_quic_cache = preemptive_quic_cache if self._disabled_svn: if HttpVersion.h11 in self._disabled_svn: raise RuntimeError( - "HTTP/1.1 cannot be disabled. It will be allowed in a future urllib3 version." + "HTTP/1.1 cannot be disabled. It will be allowed in a future major version." ) # valuable intel self.conn_info: ConnectionInfo | None = None + self._promises: list[ResponsePromise] = [] + + def __contains__(self, item: ResponsePromise) -> bool: + return item in self._promises + @property def disabled_svn(self) -> set[HttpVersion]: return self._disabled_svn @@ -225,6 +303,14 @@ def _http_vsn(self) -> int: assert self._svn is not None return int(self._svn.value.split("/")[-1].replace(".", "")) + @property + def is_saturated(self) -> bool: + raise NotImplementedError + + @property + def is_multiplexed(self) -> bool: + raise NotImplementedError + def _upgrade(self) -> None: """Upgrade conn from svn ver to max supported.""" raise NotImplementedError @@ -292,11 +378,13 @@ def endheaders( *, encode_chunked: bool = False, expect_body_afterward: bool = False, - ) -> None: + ) -> ResponsePromise | None: """This method conclude the request context construction.""" raise NotImplementedError - def getresponse(self) -> LowLevelResponse: + def getresponse( + self, *, promise: ResponsePromise | None = None + ) -> LowLevelResponse: """Fetch the HTTP response. You SHOULD not retrieve the body in that method, it SHOULD be done in the LowLevelResponse, so it enable stream capabilities and remain efficient. """ @@ -311,7 +399,7 @@ def send( data: (bytes | typing.IO[typing.Any] | typing.Iterable[bytes] | str), *, eot: bool = False, - ) -> None: + ) -> ResponsePromise | None: """The send() method SHOULD be invoked after calling endheaders() if and only if the request context specify explicitly that a body is going to be sent.""" raise NotImplementedError diff --git a/src/urllib3/backend/hface.py b/src/urllib3/backend/hface.py index 137c9f9fd8..b43f151312 100644 --- a/src/urllib3/backend/hface.py +++ b/src/urllib3/backend/hface.py @@ -3,7 +3,6 @@ import socket import sys import typing -from http.client import ResponseNotReady, responses from socket import SOCK_DGRAM, SOCK_STREAM try: # Compiled with SSL? @@ -15,6 +14,7 @@ SSLTransport = None # type: ignore from .._collections import HTTPHeaderDict +from .._constant import DEFAULT_BLOCKSIZE, responses from ..contrib.hface import ( HTTP1Protocol, HTTP2Protocol, @@ -32,7 +32,13 @@ HeadersReceived, StreamResetReceived, ) -from ..exceptions import EarlyResponse, InvalidHeader, ProtocolError, SSLError +from ..exceptions import ( + EarlyResponse, + InvalidHeader, + ProtocolError, + ResponseNotReady, + SSLError, +) from ..util import parse_alt_svc from ._base import ( BaseBackend, @@ -40,6 +46,7 @@ HttpVersion, LowLevelResponse, QuicPreemptiveCacheType, + ResponsePromise, ) if typing.TYPE_CHECKING: @@ -58,10 +65,10 @@ def __init__( port: int | None = None, timeout: int | float | None = -1, source_address: tuple[str, int] | None = None, - blocksize: int = 8192, + blocksize: int = DEFAULT_BLOCKSIZE, *, - socket_options: None - | _TYPE_SOCKET_OPTIONS = BaseBackend.default_socket_options, + socket_options: _TYPE_SOCKET_OPTIONS + | None = BaseBackend.default_socket_options, disabled_svn: set[HttpVersion] | None = None, preemptive_quic_cache: QuicPreemptiveCacheType | None = None, ): @@ -92,6 +99,16 @@ def __init__( self.__alt_authority: tuple[str, int] | None = None self.__session_ticket: typing.Any | None = None + @property + def is_saturated(self) -> bool: + if self._protocol is None: + return True + return self._protocol.is_available() is False + + @property + def is_multiplexed(self) -> bool: + return self._protocol is not None and self._protocol.multiplexed + def _new_conn(self) -> socket.socket | None: # handle if set up, quic cache capability. thus avoiding first TCP request prior to upgrade. if ( @@ -164,9 +181,9 @@ def _custom_tls( ca_cert_data: None | str | bytes = None, ssl_minimum_version: int | None = None, ssl_maximum_version: int | None = None, - cert_file: str | None = None, - key_file: str | None = None, - key_password: str | None = None, + cert_file: str | bytes | None = None, + key_file: str | bytes | None = None, + key_password: str | bytes | None = None, cert_fingerprint: str | None = None, assert_hostname: None | str | typing.Literal[False] = None, ) -> None: @@ -336,19 +353,19 @@ def _post_conn(self) -> None: ) if isinstance(self._protocol, HTTPOverQUICProtocol): - self.conn_info.certificate_der = self._protocol.getpeercert( # type: ignore[assignment] + self.conn_info.certificate_der = self._protocol.getpeercert( binary_form=True ) - self.conn_info.certificate_dict = self._protocol.getpeercert( # type: ignore[assignment] + self.conn_info.certificate_dict = self._protocol.getpeercert( binary_form=False ) self.conn_info.destination_address = self.sock.getpeername()[:2] self.conn_info.cipher = self._protocol.cipher() self.conn_info.tls_version = ssl.TLSVersion.TLSv1_3 - self.conn_info.issuer_certificate_dict = self._protocol.getissuercert( # type: ignore[assignment] + self.conn_info.issuer_certificate_dict = self._protocol.getissuercert( binary_form=False ) - self.conn_info.issuer_certificate_der = self._protocol.getissuercert( # type: ignore[assignment] + self.conn_info.issuer_certificate_der = self._protocol.getissuercert( binary_form=True ) @@ -361,7 +378,7 @@ def set_tunnel( ) -> None: if self.sock: # overly protective, checks are made higher, this is unreachable. - raise RuntimeError( # Defensive: mimic HttpConnection from http.client + raise RuntimeError( # Defensive: highly controlled, should be unreachable. "Can't set up tunnel for established connection" ) @@ -428,7 +445,9 @@ def _tunnel(self) -> None: if not tunnel_accepted: self.close() - message: str = responses[status] if status in responses else "UNKNOWN" + message: str = ( + responses[status] if status and status in responses else "Unknown" + ) raise OSError(f"Tunnel connection failed: {status} {message}") # We will re-initialize those afterward @@ -446,6 +465,7 @@ def __exchange_until( respect_end_stream_signal: bool = True, maximal_data_in_read: int | None = None, data_in_len_from: typing.Callable[[Event], int] | None = None, + stream_id: int | None = None, ) -> list[Event]: """This method simplify socket exchange in/out based on what the protocol state machine orders. Can be used for the initial handshake for instance.""" @@ -461,6 +481,7 @@ def __exchange_until( data_in_len: int = 0 events: list[Event] = [] + reshelve_events: list[Event] = [] if maximal_data_in_read == 0: # The '0' case amt is handled higher in the stack. @@ -517,6 +538,11 @@ def __exchange_until( self.sock.sendall(data_out) for event in iter(self._protocol.next_event, None): # type: Event + if stream_id is not None and hasattr(event, "stream_id"): + if event.stream_id != stream_id: + reshelve_events.append(event) + continue + if isinstance(event, ConnectionTerminated): if ( event.error_code == 400 @@ -561,9 +587,13 @@ def __exchange_until( and hasattr(event, "end_stream") ): if event.end_stream is True: + if reshelve_events: + self._protocol.reshelve(*reshelve_events) return events continue + if reshelve_events: + self._protocol.reshelve(*reshelve_events) return events def putrequest( @@ -628,7 +658,7 @@ def endheaders( *, encode_chunked: bool = False, expect_body_afterward: bool = False, - ) -> None: + ) -> ResponsePromise | None: if self.sock is None: self.connect() # type: ignore[attr-defined] @@ -666,7 +696,7 @@ def endheaders( if any(k == b":authority" for k, v in self.__headers) is False: raise ProtocolError( ( - "HfaceBackend do not support emitting HTTP requests without the `Host` header", + "urllib3.future do not support emitting HTTP requests without the `Host` header", "It was only permitted in HTTP/1.0 and prior. This implementation ship with HTTP/1.1+.", ) ) @@ -698,7 +728,16 @@ def endheaders( self.sock.sendall(self._protocol.bytes_to_send()) - def __read_st(self, __amt: int | None = None) -> tuple[bytes, bool]: + if should_end_stream: + rp = ResponsePromise(self, self._stream_id, self.__headers) + self._promises.append(rp) + return rp + + return None + + def __read_st( + self, __amt: int | None = None, __stream_id: int | None = None + ) -> tuple[bytes, bool]: """Allows us to defer the body loading after constructing the response object.""" eot = False @@ -711,24 +750,33 @@ def __read_st(self, __amt: int | None = None) -> tuple[bytes, bool]: data_in_len_from=lambda x: len(x.data) if isinstance(x, DataReceived) else 0, + stream_id=__stream_id, ) if events and events[-1].end_stream: eot = True - # probe for h3/quic if available, and remember it. - self._upgrade() + if not self._promises: + # probe for h3/quic if available, and remember it. + self._upgrade() # remote can refuse future inquiries, so no need to go further with this conn. if self._protocol and self._protocol.has_expired(): self.close() return ( - b"".join(e.data if isinstance(e, DataReceived) else b"" for e in events), + b"".join(e.data for e in events if isinstance(e, DataReceived)), eot, ) - def getresponse(self) -> LowLevelResponse: - if self.sock is None or self._protocol is None or self._svn is None: + def getresponse( + self, *, promise: ResponsePromise | None = None + ) -> LowLevelResponse: + if ( + self.sock is None + or self._protocol is None + or self._svn is None + or not self._promises + ): raise ResponseNotReady() # Defensive: Comply with http.client, actually tested but not reported? headers = HTTPHeaderDict() @@ -739,26 +787,36 @@ def getresponse(self) -> LowLevelResponse: receive_first=True, event_type_collectable=(HeadersReceived,), respect_end_stream_signal=False, + stream_id=promise.stream_id if promise else None, ) for event in events: - if isinstance(event, HeadersReceived): - for raw_header, raw_value in event.headers: - header: str = raw_header.decode("ascii") - value: str = raw_value.decode("iso-8859-1") - - # special headers that represent (usually) the HTTP response status, version and reason. - if header.startswith(":"): - if header == ":status" and value.isdigit(): - status = int(value) - continue - # this should be unreachable. - # it is designed to detect eventual changes lower in the stack. - raise ProtocolError( - f"Unhandled special header '{header}'" - ) # Defensive: + for raw_header, raw_value in event.headers: + header: str = raw_header.decode("ascii") + value: str = raw_value.decode("iso-8859-1") + + # special headers that represent (usually) the HTTP response status, version and reason. + if header.startswith(":"): + if header == ":status" and value.isdigit(): + status = int(value) + continue + # this should be unreachable. + # it is designed to detect eventual changes lower in the stack. + raise ProtocolError( # Defensive: + f"Unhandled special header '{header}'" + ) - headers.add(header, value) + headers.add(header, value) + + if promise is None: + for p in self._promises: + if p.stream_id == events[-1].stream_id: + promise = p + break + if promise is None: + raise ProtocolError( + f"Response received (stream: {events[-1].stream_id}) but no promise in-flight" + ) # this should be unreachable if status is None: @@ -769,16 +827,23 @@ def getresponse(self) -> LowLevelResponse: eot = events[-1].end_stream is True response = LowLevelResponse( - dict(self.__headers)[b":method"].decode("ascii"), + dict(promise.request_headers)[b":method"].decode("ascii"), status, self._http_vsn, - responses[status] if status in responses else "UNKNOWN", + responses[status] if status in responses else "Unknown", headers, self.__read_st if not eot else None, authority=self.host, port=self.port, + stream_id=promise.stream_id, ) + promise.response = response + response.from_promise = promise + + # we delivered a response, we can safely remove the promise from queue. + self._promises.remove(promise) + # keep last response self._response: LowLevelResponse = response @@ -787,7 +852,8 @@ def getresponse(self) -> LowLevelResponse: self.__session_ticket = self._protocol.session_ticket if eot: - self._upgrade() + if not self._promises: + self._upgrade() # remote can refuse future inquiries, so no need to go further with this conn. if self._protocol and self._protocol.has_expired(): @@ -800,7 +866,7 @@ def send( data: (bytes | typing.IO[typing.Any] | typing.Iterable[bytes] | str), *, eot: bool = False, - ) -> None: + ) -> ResponsePromise | None: """We might be receiving a chunk constructed downstream""" if self.sock is None or self._stream_id is None or self._protocol is None: # this is unreachable in normal condition as urllib3 @@ -832,8 +898,11 @@ def send( self._protocol.bytes_received(self.sock.recv(self.blocksize)) # this is a bad sign. we should stop sending and instead retrieve the response. - if self._protocol.has_pending_event(): - raise EarlyResponse() + if self._protocol.has_pending_event(stream_id=self._stream_id): + rp = ResponsePromise(self, self._stream_id, self.__headers) + self._promises.append(rp) + + raise EarlyResponse(promise=rp) if self.__remaining_body_length: self.__remaining_body_length -= len(data) @@ -853,6 +922,8 @@ def send( if _HAS_SYS_AUDIT: sys.audit("http.client.send", self, data) + remote_pipe_shutdown: BrokenPipeError | None = None + # some protocols may impose regulated frame size # so expect multiple frame per send() while True: @@ -861,13 +932,26 @@ def send( if not data_out: break - self.sock.sendall(data_out) + try: + self.sock.sendall(data_out) + except BrokenPipeError as e: + remote_pipe_shutdown = e + + if eot or remote_pipe_shutdown: + rp = ResponsePromise(self, self._stream_id, self.__headers) + self._promises.append(rp) + if remote_pipe_shutdown: + remote_pipe_shutdown.promise = rp # type: ignore[attr-defined] + raise remote_pipe_shutdown + return rp except self._protocol.exceptions() as e: raise ProtocolError( # Defensive: In the unlikely event that exception may leak from below e ) from e + return None + def close(self) -> None: if self.sock: if self._protocol is not None: diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py index f37f71ef41..07e77e26a5 100644 --- a/src/urllib3/connection.py +++ b/src/urllib3/connection.py @@ -5,8 +5,6 @@ import re import socket import typing -from http.client import HTTPException as HTTPException # noqa: F401 -from http.client import ResponseNotReady from socket import timeout as SocketTimeout if typing.TYPE_CHECKING: @@ -22,7 +20,7 @@ ProxyConfig, ) -from ._collections import HTTPHeaderDict +from ._constant import DEFAULT_BLOCKSIZE from .util.timeout import _DEFAULT_TIMEOUT, Timeout from .util.util import to_str from .util.wait import wait_for_read @@ -39,19 +37,21 @@ class BaseSSLError(BaseException): # type: ignore[no-redef] from ._version import __version__ -from .backend import HfaceBackend, HttpVersion, QuicPreemptiveCacheType +from .backend import HfaceBackend, HttpVersion, QuicPreemptiveCacheType, ResponsePromise +from .exceptions import ConnectTimeoutError, EarlyResponse +from .exceptions import HTTPError as HTTPException # noqa from .exceptions import ( - ConnectTimeoutError, - EarlyResponse, NameResolutionError, NewConnectionError, ProxyError, + ResponseNotReady, ) from .util import SKIP_HEADER, SKIPPABLE_HEADERS, connection, ssl_ from .util.request import body_to_chunks from .util.ssl_ import assert_fingerprint as _assert_fingerprint from .util.ssl_ import ( create_urllib3_context, + is_capable_for_quic, is_ipaddress, resolve_cert_reqs, resolve_ssl_version, @@ -127,9 +127,9 @@ def __init__( *, timeout: _TYPE_TIMEOUT_INTERNAL = _DEFAULT_TIMEOUT, source_address: tuple[str, int] | None = None, - blocksize: int = 8192, - socket_options: None - | _TYPE_SOCKET_OPTIONS = HfaceBackend.default_socket_options, + blocksize: int = DEFAULT_BLOCKSIZE, + socket_options: _TYPE_SOCKET_OPTIONS + | None = HfaceBackend.default_socket_options, proxy: Url | None = None, proxy_config: ProxyConfig | None = None, disabled_svn: set[HttpVersion] | None = None, @@ -149,7 +149,6 @@ def __init__( self.proxy_config = proxy_config self._has_connected_to_proxy = False - self._response_options = None @property def host(self) -> str: @@ -248,6 +247,9 @@ def is_closed(self) -> bool: def is_connected(self) -> bool: if self.sock is None: return False + # wait_for_read become flaky with concurrent streams! + if self._promises: + return True return not wait_for_read(self.sock, timeout=0.0) @property @@ -301,8 +303,6 @@ def putheader(self, header: str, *values: str) -> None: f"urllib3.util.SKIP_HEADER only supports '{skippable_headers}'" ) - # `request` method's signature intentionally violates LSP. - # urllib3's API is different from `http.client.HTTPConnection` and the subclassing is only incidental. def request( self, method: str, @@ -314,7 +314,7 @@ def request( preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, - ) -> None: + ) -> ResponsePromise: # Update the inner socket's timeout value to send the request. # This only triggers if the connection is re-used. if self.sock is not None: @@ -328,7 +328,7 @@ def request( # because sometimes we can still salvage a response # off the wire even if we aren't able to completely # send the request body. - self._response_options = _ResponseOptions( + response_options = _ResponseOptions( request_method=method, request_url=url, preload_content=preload_content, @@ -390,7 +390,12 @@ def request( if overrule_content_length and header.lower() == "content-length": value = str(content_length) self.putheader(header, value) - self.endheaders(expect_body_afterward=chunks is not None) + + rp = self.endheaders(expect_body_afterward=chunks is not None) + + if rp: + rp.set_parameter("response_options", response_options) + return rp try: # If we're given a body we start sending that in chunks. @@ -403,12 +408,21 @@ def request( if isinstance(chunk, str): chunk = chunk.encode("utf-8") self.send(chunk) - self.send(b"", eot=True) - except EarlyResponse: - pass + rp = self.send(b"", eot=True) + except EarlyResponse as e: + rp = e.promise + except BrokenPipeError as e: + rp = e.promise # type: ignore[attr-defined] + assert rp is not None + rp.set_parameter("response_options", response_options) + raise e + + assert rp is not None + rp.set_parameter("response_options", response_options) + return rp def getresponse( # type: ignore[override] - self, + self, *, promise: ResponsePromise | None = None ) -> HTTPResponse: """ Get the response from the server. @@ -418,13 +432,9 @@ def getresponse( # type: ignore[override] If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed. """ # Raise the same error as http.client.HTTPConnection - if self._response_options is None or self.sock is None: + if self.sock is None: raise ResponseNotReady() - # Reset this attribute for being used again. - resp_options = self._response_options - self._response_options = None - # Since the connection's timeout value may have been updated # we need to set the timeout on the socket. self.sock.settimeout(self.timeout) @@ -433,9 +443,15 @@ def getresponse( # type: ignore[override] from .response import HTTPResponse # Get the response from backend._base.BaseBackend - low_response = super().getresponse() + low_response = super().getresponse(promise=promise) - assert isinstance(low_response.msg, HTTPHeaderDict) + if promise is None: + promise = low_response.from_promise + + if promise is None: + raise OSError + + resp_options: _ResponseOptions = promise.get_parameter("response_options") # type: ignore[assignment] headers = low_response.msg response = HTTPResponse( @@ -471,6 +487,11 @@ class HTTPSConnection(HTTPConnection): ssl_minimum_version: int | None = None ssl_maximum_version: int | None = None assert_fingerprint: str | None = None + cert_file: str | None = None + key_file: str | None = None + key_password: str | None = None + cert_data: str | bytes | None = None + key_data: str | bytes | None = None def __init__( self, @@ -479,9 +500,9 @@ def __init__( *, timeout: _TYPE_TIMEOUT_INTERNAL = _DEFAULT_TIMEOUT, source_address: tuple[str, int] | None = None, - blocksize: int = 8192, - socket_options: None - | _TYPE_SOCKET_OPTIONS = HTTPConnection.default_socket_options, + blocksize: int = DEFAULT_BLOCKSIZE, + socket_options: _TYPE_SOCKET_OPTIONS + | None = HTTPConnection.default_socket_options, disabled_svn: set[HttpVersion] | None = None, preemptive_quic_cache: QuicPreemptiveCacheType | None = None, proxy: Url | None = None, @@ -500,35 +521,10 @@ def __init__( cert_file: str | None = None, key_file: str | None = None, key_password: str | None = None, + cert_data: str | bytes | None = None, + key_data: str | bytes | None = None, ) -> None: - # Some parameters may defacto exclude HTTP/3 over QUIC. - # Let's check all of those: - # -> TLS 1.3 required - # -> One of the three supported ciphers (listed bellow) - quic_disable: bool = False - - if ssl_context is not None: - if ( - isinstance(ssl_context.maximum_version, ssl.TLSVersion) - and ssl_context.maximum_version <= ssl.TLSVersion.TLSv1_2 - ): - quic_disable = True - else: - any_capable_cipher: bool = False - for cipher_dict in ssl_context.get_ciphers(): - if cipher_dict["name"] in [ - "TLS_AES_128_GCM_SHA256", - "TLS_AES_256_GCM_SHA384", - "TLS_CHACHA20_POLY1305_SHA256", - ]: - any_capable_cipher = True - if not any_capable_cipher: - quic_disable = True - - if ssl_maximum_version and ssl_maximum_version <= ssl.TLSVersion.TLSv1_2: - quic_disable = True - - if quic_disable: + if not is_capable_for_quic(ssl_context, ssl_maximum_version): if disabled_svn is None: disabled_svn = set() @@ -549,6 +545,8 @@ def __init__( self.key_file = key_file self.cert_file = cert_file + self.cert_data = cert_data + self.key_data = key_data self.key_password = key_password self.ssl_context = ssl_context self.server_hostname = server_hostname @@ -574,6 +572,7 @@ def connect(self) -> None: self.sock = sock = self._new_conn() try: + # the protocol/state-machine may also ship with an external TLS Engine. self._custom_tls( self.ssl_context, self.ca_certs, @@ -581,8 +580,8 @@ def connect(self) -> None: self.ca_cert_data, self.ssl_minimum_version, self.ssl_maximum_version, - self.cert_file, - self.key_file, + self.cert_file or self.cert_data, + self.key_file or self.key_data, self.key_password, self.assert_fingerprint, ) @@ -640,6 +639,8 @@ def connect(self) -> None: assert_hostname=self.assert_hostname, assert_fingerprint=self.assert_fingerprint, alpn_protocols=alpn_protocols, + cert_data=self.cert_data, + key_data=self.key_data, ) self.sock = sock_and_verified.socket # type: ignore[assignment] self.is_verified = sock_and_verified.is_verified @@ -683,6 +684,8 @@ def _connect_tls_proxy( key_password=None, tls_in_tls=False, alpn_protocols=alpn_protocols, + cert_data=None, + key_data=None, ) self.proxy_is_verified = sock_and_verified.is_verified return sock_and_verified.socket # type: ignore[return-value] @@ -717,6 +720,8 @@ def _ssl_wrap_socket_and_match_hostname( ssl_context: ssl.SSLContext | None, tls_in_tls: bool = False, alpn_protocols: list[str] | None = None, + cert_data: str | bytes | None = None, + key_data: str | bytes | None = None, ) -> _WrappedAndVerifiedSocket: """Logic for constructing an SSLContext from all TLS parameters, passing that down into ssl_wrap_socket, and then doing certificate verification @@ -783,6 +788,8 @@ def _ssl_wrap_socket_and_match_hostname( ssl_context=context, tls_in_tls=tls_in_tls, alpn_protocols=alpn_protocols, + certdata=cert_data, + keydata=key_data, ) try: diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index afaa266bac..e46e037fd5 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -13,13 +13,12 @@ from ._collections import HTTPHeaderDict from ._request_methods import RequestMethods from ._typing import _TYPE_BODY, _TYPE_BODY_POSITION, _TYPE_TIMEOUT, ProxyConfig -from .backend import ConnectionInfo +from .backend import ConnectionInfo, HttpVersion, ResponsePromise from .connection import ( BaseSSLError, BrokenPipeError, DummyConnection, HTTPConnection, - HTTPException, HTTPSConnection, _wrap_proxy_error, ) @@ -36,10 +35,11 @@ ProtocolError, ProxyError, ReadTimeoutError, + ResponseNotReady, SSLError, TimeoutError, ) -from .response import BaseHTTPResponse +from .response import HTTPResponse from .util.connection import is_connection_dropped from .util.proxy import connection_requires_http_tunnel from .util.request import NOT_FORWARDABLE_HEADERS, set_file_position @@ -251,7 +251,9 @@ def _new_conn(self) -> HTTPConnection: ) return conn - def _get_conn(self, timeout: float | None = None) -> HTTPConnection: + def _get_conn( + self, timeout: float | None = None, no_new: bool = False + ) -> HTTPConnection: """ Get a connection. Will return a pooled connection if one is available. @@ -270,7 +272,6 @@ def _get_conn(self, timeout: float | None = None) -> HTTPConnection: try: conn = self.pool.get(block=self.block, timeout=timeout) - except AttributeError: # self.pool is None raise ClosedPoolError(self, "Pool is closed.") from None # Defensive: @@ -282,6 +283,9 @@ def _get_conn(self, timeout: float | None = None) -> HTTPConnection: ) from None pass # Oh well, we'll create a new connection then + if no_new and conn is None: + raise ValueError + # If this is a persistent connection, check if it got disconnected if conn and is_connection_dropped(conn): log.debug("Resetting dropped connection: %s", self.host) @@ -374,6 +378,212 @@ def _raise_timeout( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err + def get_response( + self, *, promise: ResponsePromise | None = None + ) -> HTTPResponse | None: + """ + Retrieve the first response available in the pool. + This method should be called after issuing at least one request with ``multiplexed=True``. + If none available, return None. + """ + connections = [] + response = None + + while True: + try: + conn = self._get_conn(no_new=True) + except ValueError: + break + + connections.append(conn) + + if promise: + if promise in conn: + response = conn.getresponse(promise=promise) + else: + continue + else: + try: + response = conn.getresponse() + except ResponseNotReady: + continue + break + + for conn in connections: + self._put_conn(conn) + + if promise is not None and response is None: + raise ValueError + + if response is None: + return None + + from_promise = None + + if promise: + from_promise = promise + else: + if ( + response._fp + and hasattr(response._fp, "from_promise") + and response._fp.from_promise + ): + from_promise = response._fp.from_promise + + if from_promise is None: + raise ValueError + + # Retrieve request ctx + method = typing.cast(str, from_promise.get_parameter("method")) + url = typing.cast(str, from_promise.get_parameter("url")) + body = typing.cast( + typing.Optional[_TYPE_BODY], from_promise.get_parameter("body") + ) + headers = typing.cast(HTTPHeaderDict, from_promise.get_parameter("headers")) + retries = typing.cast(Retry, from_promise.get_parameter("retries")) + preload_content = typing.cast( + bool, from_promise.get_parameter("preload_content") + ) + decode_content = typing.cast(bool, from_promise.get_parameter("decode_content")) + timeout = typing.cast( + typing.Optional[_TYPE_TIMEOUT], from_promise.get_parameter("timeout") + ) + redirect = typing.cast(bool, from_promise.get_parameter("redirect")) + assert_same_host = typing.cast( + bool, from_promise.get_parameter("assert_same_host") + ) + pool_timeout = from_promise.get_parameter("pool_timeout") + response_kw = typing.cast( + typing.MutableMapping[str, typing.Any], + from_promise.get_parameter("response_kw"), + ) + chunked = typing.cast(bool, from_promise.get_parameter("chunked")) + body_pos = typing.cast( + _TYPE_BODY_POSITION, from_promise.get_parameter("body_pos") + ) + + # Handle redirect? + redirect_location = redirect and response.get_redirect_location() + if redirect_location: + if response.status == 303: + method = "GET" + body = None + headers = HTTPHeaderDict(headers) + + for should_be_removed_header in NOT_FORWARDABLE_HEADERS: + headers.discard(should_be_removed_header) + + try: + retries = retries.increment(method, url, response=response, _pool=self) + except MaxRetryError: + if retries.raise_on_redirect: + response.drain_conn() + raise + return response + + response.drain_conn() + retries.sleep_for_retry(response) + log.debug("Redirecting %s -> %s", url, redirect_location) + new_promise = self.urlopen( + method, + redirect_location, + body, + headers, + retries=retries, + redirect=redirect, + assert_same_host=assert_same_host, + timeout=timeout, + pool_timeout=pool_timeout, + release_conn=True, + chunked=chunked, + body_pos=body_pos, + preload_content=preload_content, + decode_content=decode_content, + multiplexed=True, + **response_kw, + ) + + return self.get_response(promise=new_promise if promise else None) + + # Check if we should retry the HTTP response. + has_retry_after = bool(response.headers.get("Retry-After")) + if retries.is_retry(method, response.status, has_retry_after): + try: + retries = retries.increment(method, url, response=response, _pool=self) + except MaxRetryError: + if retries.raise_on_status: + response.drain_conn() + raise + return response + + response.drain_conn() + retries.sleep(response) + log.debug("Retry: %s", url) + new_promise = self.urlopen( + method, + url, + body, + headers, + retries=retries, + redirect=redirect, + assert_same_host=assert_same_host, + timeout=timeout, + pool_timeout=pool_timeout, + release_conn=False, + chunked=chunked, + body_pos=body_pos, + preload_content=preload_content, + decode_content=decode_content, + multiplexed=True, + **response_kw, + ) + + return self.get_response(promise=new_promise if promise else None) + + return response + + @typing.overload + def _make_request( + self, + conn: HTTPConnection, + method: str, + url: str, + body: _TYPE_BODY | None = ..., + headers: typing.Mapping[str, str] | None = ..., + retries: Retry | None = ..., + timeout: _TYPE_TIMEOUT = ..., + chunked: bool = ..., + response_conn: HTTPConnection | None = ..., + preload_content: bool = ..., + decode_content: bool = ..., + enforce_content_length: bool = ..., + on_post_connection: typing.Callable[[ConnectionInfo], None] | None = ..., + *, + multiplexed: Literal[True], + ) -> ResponsePromise: + ... + + @typing.overload + def _make_request( + self, + conn: HTTPConnection, + method: str, + url: str, + body: _TYPE_BODY | None = ..., + headers: typing.Mapping[str, str] | None = ..., + retries: Retry | None = ..., + timeout: _TYPE_TIMEOUT = ..., + chunked: bool = ..., + response_conn: HTTPConnection | None = ..., + preload_content: bool = ..., + decode_content: bool = ..., + enforce_content_length: bool = ..., + on_post_connection: typing.Callable[[ConnectionInfo], None] | None = ..., + *, + multiplexed: Literal[False] = ..., + ) -> HTTPResponse: + ... + def _make_request( self, conn: HTTPConnection, @@ -389,7 +599,8 @@ def _make_request( decode_content: bool = True, enforce_content_length: bool = True, on_post_connection: typing.Callable[[ConnectionInfo], None] | None = None, - ) -> BaseHTTPResponse: + multiplexed: Literal[False] | Literal[True] = False, + ) -> HTTPResponse | ResponsePromise: """ Perform a request on a given urllib connection object taken from our pool. @@ -491,10 +702,12 @@ def _make_request( if on_post_connection is not None and conn.conn_info is not None: on_post_connection(conn.conn_info) - # conn.request() calls http.client.*.request, not the method in - # urllib3.request. It also calls makefile (recv) on the socket. + if conn.is_multiplexed is False and multiplexed is True: + # overruling + multiplexed = False + try: - conn.request( + rp = conn.request( method, url, body=body, @@ -504,13 +717,13 @@ def _make_request( decode_content=decode_content, enforce_content_length=enforce_content_length, ) - # We are swallowing BrokenPipeError (errno.EPIPE) since the server is # legitimately able to close the connection after sending a valid response. # With this behaviour, the received response is still readable. - except BrokenPipeError: - pass + except BrokenPipeError as e: + rp = e.promise # type: ignore except OSError as e: + rp = None # MacOS/Linux # EPROTOTYPE is needed on macOS # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ @@ -520,6 +733,12 @@ def _make_request( # Reset the timeout for the recv() on the socket read_timeout = timeout_obj.read_timeout + if multiplexed: + if rp is None: + raise OSError + rp.set_parameter("read_timeout", read_timeout) + return rp + if not conn.is_closed: # In Python 3 socket.py will catch EAGAIN and return None when you # try and read into the file pointer created by http.client, which @@ -593,7 +812,55 @@ def is_same_host(self, url: str) -> bool: return (scheme, host, port) == (self.scheme, self.host, self.port) - def urlopen( # type: ignore[override] + @typing.overload # type: ignore[override] + def urlopen( + self, + method: str, + url: str, + body: _TYPE_BODY | None = ..., + headers: typing.Mapping[str, str] | None = ..., + retries: Retry | bool | int | None = ..., + redirect: bool = ..., + assert_same_host: bool = ..., + timeout: _TYPE_TIMEOUT = ..., + pool_timeout: int | None = ..., + release_conn: bool | None = ..., + chunked: bool = ..., + body_pos: _TYPE_BODY_POSITION | None = ..., + preload_content: bool = ..., + decode_content: bool = ..., + on_post_connection: typing.Callable[[ConnectionInfo], None] | None = ..., + *, + multiplexed: Literal[False] = ..., + **response_kw: typing.Any, + ) -> HTTPResponse: + ... + + @typing.overload + def urlopen( + self, + method: str, + url: str, + body: _TYPE_BODY | None = ..., + headers: typing.Mapping[str, str] | None = ..., + retries: Retry | bool | int | None = ..., + redirect: bool = ..., + assert_same_host: bool = ..., + timeout: _TYPE_TIMEOUT = ..., + pool_timeout: int | None = ..., + release_conn: bool | None = ..., + chunked: bool = ..., + body_pos: _TYPE_BODY_POSITION | None = ..., + preload_content: bool = ..., + decode_content: bool = ..., + on_post_connection: typing.Callable[[ConnectionInfo], None] | None = ..., + *, + multiplexed: Literal[True], + **response_kw: typing.Any, + ) -> ResponsePromise: + ... + + def urlopen( self, method: str, url: str, @@ -610,8 +877,9 @@ def urlopen( # type: ignore[override] preload_content: bool = True, decode_content: bool = True, on_post_connection: typing.Callable[[ConnectionInfo], None] | None = None, + multiplexed: bool = False, **response_kw: typing.Any, - ) -> BaseHTTPResponse: + ) -> HTTPResponse | ResponsePromise: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all @@ -706,6 +974,11 @@ def urlopen( # type: ignore[override] Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. + + :param multiplexed: + Dispatch the request in a non-blocking way, this means that the + response will be retrieved in the future with the get_response() + method. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme @@ -789,22 +1062,44 @@ def urlopen( # type: ignore[override] response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object - response = self._make_request( + response = self._make_request( # type: ignore[call-overload,misc] conn, method, url, - timeout=timeout_obj, body=body, headers=headers, - chunked=chunked, retries=retries, + timeout=timeout_obj, + chunked=chunked, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, + enforce_content_length=True, on_post_connection=on_post_connection, - **response_kw, + multiplexed=multiplexed, ) + # it was established a non-multiplexed connection. fallback to original behavior. + if not isinstance(response, ResponsePromise): + multiplexed = False + + if multiplexed: + response.set_parameter("method", method) + response.set_parameter("url", url) + response.set_parameter("body", body) + response.set_parameter("headers", headers) + response.set_parameter("retries", retries) + response.set_parameter("preload_content", preload_content) + response.set_parameter("decode_content", decode_content) + response.set_parameter("timeout", timeout_obj) + response.set_parameter("redirect", redirect) + response.set_parameter("response_kw", response_kw) + response.set_parameter("pool_timeout", pool_timeout) + response.set_parameter("assert_same_host", assert_same_host) + response.set_parameter("chunked", chunked) + response.set_parameter("body_pos", body_pos) + release_this_conn = True if not conn.is_saturated else False + # Everything went great! clean_exit = True @@ -816,7 +1111,6 @@ def urlopen( # type: ignore[override] except ( TimeoutError, - HTTPException, OSError, ProtocolError, BaseSSLError, @@ -837,11 +1131,10 @@ def urlopen( # type: ignore[override] NewConnectionError, TimeoutError, SSLError, - HTTPException, ), ) and (conn and conn.proxy and not conn.has_connected_to_proxy): new_e = _wrap_proxy_error(new_e, conn.proxy.scheme) - elif isinstance(new_e, (OSError, HTTPException)): + elif isinstance(new_e, OSError): new_e = ProtocolError("Connection aborted.", new_e) retries = retries.increment( @@ -862,8 +1155,16 @@ def urlopen( # type: ignore[override] conn.close() conn = None release_this_conn = True + elif ( + conn + and conn.is_saturated is False + and conn.conn_info is not None + and conn.conn_info.http_version != HttpVersion.h11 + ): + # multiplexing allows us to issue more requests. + release_this_conn = True - if release_this_conn: + if release_this_conn is True: # Put the connection back to be reused. If the connection is # expired then it will be None, which will get replaced with a # fresh connection during _get_conn. @@ -874,7 +1175,7 @@ def urlopen( # type: ignore[override] log.warning( "Retrying (%r) after connection broken by '%r': %s", retries, err, url ) - return self.urlopen( + return self.urlopen( # type: ignore[no-any-return,call-overload,misc] method, url, body, @@ -889,12 +1190,20 @@ def urlopen( # type: ignore[override] body_pos=body_pos, preload_content=preload_content, decode_content=decode_content, + multiplexed=multiplexed, **response_kw, ) - # Handle redirect? - redirect_location = redirect and response.get_redirect_location() - if redirect_location: + if multiplexed: + assert isinstance(response, ResponsePromise) + return response # actually a response promise! + + assert isinstance(response, HTTPResponse) + + if redirect and response.get_redirect_location(): + # Handle redirect? + redirect_location = response.get_redirect_location() + if response.status == 303: method = "GET" body = None @@ -914,11 +1223,11 @@ def urlopen( # type: ignore[override] response.drain_conn() retries.sleep_for_retry(response) log.debug("Redirecting %s -> %s", url, redirect_location) - return self.urlopen( + return self.urlopen( # type: ignore[call-overload,no-any-return,misc] method, redirect_location, - body, - headers, + body=body, + headers=headers, retries=retries, redirect=redirect, assert_same_host=assert_same_host, @@ -929,6 +1238,7 @@ def urlopen( # type: ignore[override] body_pos=body_pos, preload_content=preload_content, decode_content=decode_content, + multiplexed=False, **response_kw, ) @@ -961,6 +1271,7 @@ def urlopen( # type: ignore[override] body_pos=body_pos, preload_content=preload_content, decode_content=decode_content, + multiplexed=False, **response_kw, ) @@ -1007,6 +1318,8 @@ def __init__( assert_fingerprint: str | None = None, ca_cert_dir: str | None = None, ca_cert_data: None | str | bytes = None, + cert_data: str | bytes | None = None, + key_data: str | bytes | None = None, **conn_kw: typing.Any, ) -> None: super().__init__( @@ -1029,6 +1342,8 @@ def __init__( self.ca_certs = ca_certs self.ca_cert_dir = ca_cert_dir self.ca_cert_data = ca_cert_data + self.cert_data = cert_data + self.key_data = key_data self.ssl_version = ssl_version self.ssl_minimum_version = ssl_minimum_version self.ssl_maximum_version = ssl_maximum_version @@ -1089,6 +1404,8 @@ def _new_conn(self) -> HTTPSConnection: ssl_version=self.ssl_version, ssl_minimum_version=self.ssl_minimum_version, ssl_maximum_version=self.ssl_maximum_version, + cert_data=self.cert_data, + key_data=self.key_data, **self.conn_kw, ) diff --git a/src/urllib3/contrib/hface/_configuration.py b/src/urllib3/contrib/hface/_configuration.py index cf93bdcde0..fbfdb88684 100644 --- a/src/urllib3/contrib/hface/_configuration.py +++ b/src/urllib3/contrib/hface/_configuration.py @@ -37,9 +37,9 @@ class QuicTLSConfig: cadata: bytes | None = None #: If provided, will trigger an additional load_cert_chain() upon the QUIC Configuration - certfile: str | None = None + certfile: str | bytes | None = None - keyfile: str | None = None + keyfile: str | bytes | None = None keypassword: str | bytes | None = None diff --git a/src/urllib3/contrib/hface/protocols/_protocols.py b/src/urllib3/contrib/hface/protocols/_protocols.py index 9b307e295d..4610b0dec9 100644 --- a/src/urllib3/contrib/hface/protocols/_protocols.py +++ b/src/urllib3/contrib/hface/protocols/_protocols.py @@ -14,9 +14,13 @@ from __future__ import annotations +import typing from abc import ABCMeta, abstractmethod from typing import Any, Sequence +if typing.TYPE_CHECKING: + from typing_extensions import Literal + from .._error_codes import HTTPErrorCodes from .._typing import HeadersType from ..events import Event @@ -121,10 +125,28 @@ def connection_ids(self) -> Sequence[bytes]: def session_ticket(self) -> Any | None: raise NotImplementedError + @typing.overload + def getpeercert(self, *, binary_form: Literal[True]) -> bytes: + ... + + @typing.overload + def getpeercert(self, *, binary_form: Literal[False] = ...) -> dict[str, Any]: + ... + @abstractmethod def getpeercert(self, *, binary_form: bool = False) -> bytes | dict[str, Any]: raise NotImplementedError + @typing.overload + def getissuercert(self, *, binary_form: Literal[True]) -> bytes | None: + ... + + @typing.overload + def getissuercert( + self, *, binary_form: Literal[False] = ... + ) -> dict[str, Any] | None: + ... + @abstractmethod def getissuercert( self, *, binary_form: bool = False @@ -261,10 +283,15 @@ def next_event(self) -> Event | None: raise NotImplementedError @abstractmethod - def has_pending_event(self) -> bool: + def has_pending_event(self, *, stream_id: int | None = None) -> bool: """Verify if there is queued event waiting to be consumed.""" raise NotImplementedError + @abstractmethod + def reshelve(self, *events: Event) -> None: + """Put back events into the deque.""" + raise NotImplementedError + class HTTPOverTCPProtocol(HTTPProtocol, OverTCPProtocol): """ diff --git a/src/urllib3/contrib/hface/protocols/http1/_h11.py b/src/urllib3/contrib/hface/protocols/http1/_h11.py index 1de28f3c3a..f9942044c3 100644 --- a/src/urllib3/contrib/hface/protocols/http1/_h11.py +++ b/src/urllib3/contrib/hface/protocols/http1/_h11.py @@ -223,7 +223,7 @@ def next_event(self) -> Event | None: return None return self._event_buffer.popleft() - def has_pending_event(self) -> bool: + def has_pending_event(self, *, stream_id: int | None = None) -> bool: return len(self._event_buffer) > 0 def _h11_submit(self, h11_event: h11.Event) -> None: @@ -293,3 +293,6 @@ def _maybe_start_next_cycle(self) -> None: if data: self._event_buffer.append(DataReceived(self._current_stream_id, data)) self._switched = True + + def reshelve(self, *events: Event) -> None: + raise NotImplementedError("HTTP/1.1 is not multiplexed") diff --git a/src/urllib3/contrib/hface/protocols/http2/_h2.py b/src/urllib3/contrib/hface/protocols/http2/_h2.py index 246a96cb28..a2bbbf3b09 100644 --- a/src/urllib3/contrib/hface/protocols/http2/_h2.py +++ b/src/urllib3/contrib/hface/protocols/http2/_h2.py @@ -99,7 +99,7 @@ def next_event(self) -> Event | None: return None return self._events.popleft() - def has_pending_event(self) -> bool: + def has_pending_event(self, *, stream_id: int | None = None) -> bool: return len(self._events) > 0 def _map_events(self, h2_events: list[h2.events.Event]) -> Iterator[Event]: @@ -171,3 +171,7 @@ def should_wait_remote_flow_control( return flow_remaining_bytes == 0 return amt > flow_remaining_bytes + + def reshelve(self, *events: Event) -> None: + for ev in reversed(events): + self._events.appendleft(ev) diff --git a/src/urllib3/contrib/hface/protocols/http3/_qh3.py b/src/urllib3/contrib/hface/protocols/http3/_qh3.py index 1d76c0fd88..98e0ba0e44 100644 --- a/src/urllib3/contrib/hface/protocols/http3/_qh3.py +++ b/src/urllib3/contrib/hface/protocols/http3/_qh3.py @@ -19,7 +19,10 @@ from collections import deque from os import environ from time import monotonic -from typing import Iterable, Sequence +from typing import Any, Iterable, Sequence + +if typing.TYPE_CHECKING: + from typing_extensions import Literal import qh3.h3.events as h3_events import qh3.quic.events as quic_events @@ -80,15 +83,15 @@ def __init__( if len(chosen_ciphers) == 0: raise ValueError( f"Unable to find a compatible cipher in '{tls_config.ciphers}' to establish a QUIC connection. " - f"QUIC support one of '{['TLS_'+e for e in available_ciphers.keys()]}' only." + f"QUIC support one of '{['TLS_' + e for e in available_ciphers.keys()]}' only." ) self._configuration.cipher_suites = chosen_ciphers if tls_config.certfile: self._configuration.load_cert_chain( - tls_config.certfile, # type: ignore[arg-type] - tls_config.keyfile, # type: ignore[arg-type] + tls_config.certfile, + tls_config.keyfile, tls_config.keypassword, ) @@ -108,6 +111,7 @@ def exceptions() -> tuple[type[BaseException], ...]: return ProtocolError, H3Error, QuicConnectionError, AssertionError def is_available(self) -> bool: + # todo: decide a limit. 250? return not self._terminated def has_expired(self) -> bool: @@ -155,7 +159,7 @@ def next_event(self) -> Event | None: return None return self._event_buffer.popleft() - def has_pending_event(self) -> bool: + def has_pending_event(self, *, stream_id: int | None = None) -> bool: return len(self._event_buffer) > 0 @property @@ -240,6 +244,16 @@ def should_wait_remote_flow_control( ) -> bool | None: return self._data_in_flight + @typing.overload + def getissuercert(self, *, binary_form: Literal[True]) -> bytes | None: + ... + + @typing.overload + def getissuercert( + self, *, binary_form: Literal[False] = ... + ) -> dict[str, Any] | None: + ... + def getissuercert( self, *, binary_form: bool = False ) -> bytes | dict[str, typing.Any] | None: @@ -317,6 +331,14 @@ def getissuercert( return issuer_info + @typing.overload + def getpeercert(self, *, binary_form: Literal[True]) -> bytes: + ... + + @typing.overload + def getpeercert(self, *, binary_form: Literal[False] = ...) -> dict[str, Any]: + ... + def getpeercert( self, *, binary_form: bool = False ) -> bytes | dict[str, typing.Any]: @@ -471,3 +493,7 @@ def cipher(self) -> str | None: raise ValueError("TLS handshake has not been done yet") return f"TLS_{cipher_suite.name}" + + def reshelve(self, *events: Event) -> None: + for ev in reversed(events): + self._event_buffer.appendleft(ev) diff --git a/src/urllib3/contrib/imcc/__init__.py b/src/urllib3/contrib/imcc/__init__.py new file mode 100644 index 0000000000..d3f58da008 --- /dev/null +++ b/src/urllib3/contrib/imcc/__init__.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import os +import secrets +import stat +import sys +import typing +import warnings +from hashlib import sha256 +from io import UnsupportedOperation + +if typing.TYPE_CHECKING: + import ssl + + +def load_cert_chain( + ctx: ssl.SSLContext, + certdata: str | bytes, + keydata: str | bytes | None = None, + password: typing.Callable[[], str | bytes] | str | bytes | None = None, +) -> None: + """ + Unique workaround the known limitation of CPython inability to initialize the mTLS context without files. + Only supported on Linux, FreeBSD, and OpenBSD. + :raise UnsupportedOperation: If anything goes wrong in the process. + """ + if ( + sys.platform != "linux" + and sys.platform.startswith("freebsd") is False + and sys.platform.startswith("openbsd") is False + ): + raise UnsupportedOperation( + f"Unable to provide support for in-memory client certificate: Unsupported platform {sys.platform}" + ) + + unique_name: str = f"{sha256(secrets.token_bytes(32)).hexdigest()}.pem" + + if isinstance(certdata, bytes): + certdata = certdata.decode("ascii") + + if keydata is not None: + if isinstance(keydata, bytes): + keydata = keydata.decode("ascii") + + if hasattr(os, "memfd_create"): + fd = os.memfd_create(unique_name, os.MFD_CLOEXEC) + else: + # this branch patch is for CPython <3.8 and PyPy 3.7+ + from ctypes import c_int, c_ushort, cdll, create_string_buffer, get_errno, util + + loc = util.find_library("rt") or util.find_library("c") + + if not loc: + raise UnsupportedOperation( + "Unable to provide support for in-memory client certificate: libc or librt not found." + ) + + lib = cdll.LoadLibrary(loc) + + _shm_open = lib.shm_open + # _shm_unlink = lib.shm_unlink + + buf_name = create_string_buffer(unique_name.encode()) + + fd = _shm_open( + buf_name, + c_int(os.O_RDWR | os.O_CREAT), + c_ushort(stat.S_IRUSR | stat.S_IWUSR), + ) + + if fd == -1: + raise UnsupportedOperation( + f"Unable to provide support for in-memory client certificate: {os.strerror(get_errno())}" + ) + + # Linux 3.17+ + path = f"/proc/self/fd/{fd}" + + # Alt-path + shm_path = f"/dev/shm/{unique_name}" + + if os.path.exists(path) is False: + if os.path.exists(shm_path): + path = shm_path + else: + os.fdopen(fd).close() + + raise UnsupportedOperation( + "Unable to provide support for in-memory client certificate: no virtual patch available?" + ) + + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + + with open(path, "w") as fp: + fp.write(certdata) + + if keydata: + fp.write(keydata) + + path = fp.name + + ctx.load_cert_chain(path, password=password) + + # we shall start cleaning remnants + os.fdopen(fd).close() + + if os.path.exists(shm_path): + os.unlink(shm_path) + + if os.path.exists(path) or os.path.exists(shm_path): + warnings.warn( + "In-memory client certificate: The kernel leaked a file descriptor outside of its expected lifetime.", + ResourceWarning, + ) + + +__all__ = ("load_cert_chain",) diff --git a/src/urllib3/contrib/pyopenssl.py b/src/urllib3/contrib/pyopenssl.py index 67e55473e4..a492c31cf4 100644 --- a/src/urllib3/contrib/pyopenssl.py +++ b/src/urllib3/contrib/pyopenssl.py @@ -3,9 +3,9 @@ import warnings warnings.warn( - "'urllib3.contrib.pyopenssl' module has been removed " - "in urllib3 v2.1.0 due to incompatibilities with our QUIC integration." - "While the import still work, it is rendered completely ineffective.", + """'urllib3.contrib.pyopenssl' module has been removed in urllib3.future due to incompatibilities with our QUIC integration. +While the import still work, it is rendered completely ineffective. Were you looking for in-memory client certificate? +It is natively supported since v2.2, check the documentation.""", category=DeprecationWarning, stacklevel=2, ) diff --git a/src/urllib3/exceptions.py b/src/urllib3/exceptions.py index 6698be6ab7..a79550a122 100644 --- a/src/urllib3/exceptions.py +++ b/src/urllib3/exceptions.py @@ -3,10 +3,10 @@ import socket import typing from email.errors import MessageDefect -from http.client import IncompleteRead as httplib_IncompleteRead if typing.TYPE_CHECKING: from ._typing import _TYPE_REDUCE_RESULT + from .backend import ResponsePromise from .connection import HTTPConnection from .connectionpool import ConnectionPool from .response import HTTPResponse @@ -228,7 +228,7 @@ class BodyNotHttplibCompatible(HTTPError): """ -class IncompleteRead(HTTPError, httplib_IncompleteRead): +class IncompleteRead(ProtocolError): """ Response length doesn't match expected Content-Length @@ -237,21 +237,21 @@ class IncompleteRead(HTTPError, httplib_IncompleteRead): """ def __init__(self, partial: int, expected: int) -> None: - self.partial = partial # type: ignore[assignment] + self.partial = partial self.expected = expected def __repr__(self) -> str: return "IncompleteRead(%i bytes read, %i more expected)" % ( - self.partial, # type: ignore[str-format] + self.partial, self.expected, ) -class InvalidChunkLength(HTTPError, httplib_IncompleteRead): +class InvalidChunkLength(ProtocolError): """Invalid chunk length in a chunked response.""" def __init__(self, response: HTTPResponse, length: bytes) -> None: - self.partial: int = response.tell() # type: ignore[assignment] + self.partial: int = response.tell() self.expected: int | None = response.length_remaining self.response = response self.length = length @@ -304,3 +304,10 @@ class UnrewindableBodyError(HTTPError): class EarlyResponse(HTTPError): """urllib3 received a response prior to sending the whole body""" + + def __init__(self, promise: ResponsePromise) -> None: + self.promise = promise + + +class ResponseNotReady(HTTPError): + """Kept for BC""" diff --git a/src/urllib3/poolmanager.py b/src/urllib3/poolmanager.py index bdab1f2e5d..bfa8d72d70 100644 --- a/src/urllib3/poolmanager.py +++ b/src/urllib3/poolmanager.py @@ -8,9 +8,16 @@ from urllib.parse import urljoin from ._collections import HTTPHeaderDict, RecentlyUsedContainer +from ._constant import DEFAULT_BLOCKSIZE from ._request_methods import RequestMethods -from ._typing import _TYPE_SOCKET_OPTIONS, ProxyConfig -from .backend import HttpVersion, QuicPreemptiveCacheType +from ._typing import ( + _TYPE_BODY, + _TYPE_BODY_POSITION, + _TYPE_SOCKET_OPTIONS, + _TYPE_TIMEOUT, + ProxyConfig, +) +from .backend import HttpVersion, QuicPreemptiveCacheType, ResponsePromise from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme from .exceptions import ( LocationValueError, @@ -18,7 +25,7 @@ ProxySchemeUnknown, URLSchemeUnknown, ) -from .response import BaseHTTPResponse +from .response import HTTPResponse from .util.proxy import connection_requires_http_tunnel from .util.request import NOT_FORWARDABLE_HEADERS from .util.retry import Retry @@ -48,10 +55,9 @@ "ssl_context", "key_password", "server_hostname", + "cert_data", + "key_data", ) -# Default value for `blocksize` - a new parameter introduced to -# http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7 -_DEFAULT_BLOCKSIZE = 16384 _SelfT = typing.TypeVar("_SelfT") @@ -94,6 +100,9 @@ class PoolKey(typing.NamedTuple): key_server_hostname: str | None key_blocksize: int | None key_disabled_svn: set[HttpVersion] | None + key_cert_data: str | bytes | None + key_key_data: str | bytes | None + key_multiplexed: bool def _default_key_normalizer( @@ -146,9 +155,9 @@ def _default_key_normalizer( if field not in context: context[field] = None - # Default key_blocksize to _DEFAULT_BLOCKSIZE if missing from the context + # Default key_blocksize to DEFAULT_BLOCKSIZE if missing from the context if context.get("key_blocksize") is None: - context["key_blocksize"] = _DEFAULT_BLOCKSIZE + context["key_blocksize"] = DEFAULT_BLOCKSIZE return key_class(**context) @@ -255,10 +264,10 @@ def _new_pool( if request_context is None: request_context = self.connection_pool_kw.copy() - # Default blocksize to _DEFAULT_BLOCKSIZE if missing or explicitly + # Default blocksize to DEFAULT_BLOCKSIZE if missing or explicitly # set to 'None' in the request_context. if request_context.get("blocksize") is None: - request_context["blocksize"] = _DEFAULT_BLOCKSIZE + request_context["blocksize"] = DEFAULT_BLOCKSIZE # Although the context has everything necessary to create the pool, # this function has historically only used the scheme, host, and port @@ -415,9 +424,198 @@ def _proxy_requires_url_absolute_form(self, parsed_url: Url) -> bool: self.proxy, self.proxy_config, parsed_url.scheme ) - def urlopen( # type: ignore[override] + def get_response( + self, *, promise: ResponsePromise | None = None + ) -> HTTPResponse | None: + """ + Retrieve the first response available in the pools. + This method should be called after issuing at least one request with ``multiplexed=True``. + If none available, return None. + """ + put_back_pool = [] + response = None + + for pool_key in self.pools.keys(): + pool: HTTPConnectionPool | None = self.pools.get(pool_key) + + if not pool: + continue + + response = pool.get_response(promise=promise) + put_back_pool.append((pool_key, pool)) + + if response: + break + + for pool_key, pool in put_back_pool: + self.pools[pool_key] = pool + + if response is None: + return None + + from_promise = None + + if promise: + from_promise = promise + else: + if ( + response._fp + and hasattr(response._fp, "from_promise") + and response._fp.from_promise + ): + from_promise = response._fp.from_promise + + if from_promise is None: + raise ValueError + + # Retrieve request ctx + method = typing.cast(str, from_promise.get_parameter("method")) + url = typing.cast(str, from_promise.get_parameter("pm_url")) + body = typing.cast( + typing.Union[_TYPE_BODY, None], from_promise.get_parameter("body") + ) + headers = typing.cast( + typing.Union[HTTPHeaderDict, None], from_promise.get_parameter("headers") + ) + retries = typing.cast(Retry, from_promise.get_parameter("retries")) + preload_content = typing.cast( + bool, from_promise.get_parameter("preload_content") + ) + decode_content = typing.cast(bool, from_promise.get_parameter("decode_content")) + timeout = typing.cast( + typing.Union[_TYPE_TIMEOUT, None], from_promise.get_parameter("timeout") + ) + redirect = typing.cast(bool, from_promise.get_parameter("pm_redirect")) + assert_same_host = typing.cast( + bool, from_promise.get_parameter("assert_same_host") + ) + pool_timeout = from_promise.get_parameter("pool_timeout") + response_kw = typing.cast( + typing.MutableMapping[str, typing.Any], + from_promise.get_parameter("response_kw"), + ) + chunked = typing.cast(bool, from_promise.get_parameter("chunked")) + body_pos = typing.cast( + _TYPE_BODY_POSITION, from_promise.get_parameter("body_pos") + ) + + # Handle redirect? + if redirect and response.get_redirect_location(): + redirect_location = response.get_redirect_location() + assert isinstance(redirect_location, str) + + if response.status == 303: + method = "GET" + body = None + headers = HTTPHeaderDict(headers) + + for should_be_removed_header in NOT_FORWARDABLE_HEADERS: + headers.discard(should_be_removed_header) + + try: + retries = retries.increment( + method, url, response=response, _pool=response._pool + ) + except MaxRetryError: + if retries.raise_on_redirect: + response.drain_conn() + raise + return response + + response.drain_conn() + retries.sleep_for_retry(response) + log.debug("Redirecting %s -> %s", url, redirect_location) + + new_promise = self.urlopen( + method, + urljoin(url, redirect_location), + True, + body=body, + headers=headers, + retries=retries, + assert_same_host=assert_same_host, + timeout=timeout, + pool_timeout=pool_timeout, + release_conn=True, + chunked=chunked, + body_pos=body_pos, + preload_content=preload_content, + decode_content=decode_content, + multiplexed=True, + **response_kw, + ) + + return self.get_response(promise=new_promise if promise else None) + + # Check if we should retry the HTTP response. + has_retry_after = bool(response.headers.get("Retry-After")) + if retries.is_retry(method, response.status, has_retry_after): + redirect_location = response.get_redirect_location() + assert isinstance(redirect_location, str) + + try: + retries = retries.increment( + method, url, response=response, _pool=response._pool + ) + except MaxRetryError: + if retries.raise_on_status: + response.drain_conn() + raise + return response + + response.drain_conn() + retries.sleep(response) + log.debug("Retry: %s", url) + new_promise = self.urlopen( + method, + urljoin(url, redirect_location), + True, + body=body, + headers=headers, + retries=retries, + assert_same_host=assert_same_host, + timeout=timeout, + pool_timeout=pool_timeout, + release_conn=False, + chunked=chunked, + body_pos=body_pos, + preload_content=preload_content, + decode_content=decode_content, + multiplexed=True, + **response_kw, + ) + + return self.get_response(promise=new_promise if promise else None) + + return response + + @typing.overload # type: ignore[override] + def urlopen( + self, + method: str, + url: str, + redirect: bool = True, + *, + multiplexed: Literal[False] = ..., + **kw: typing.Any, + ) -> HTTPResponse: + ... + + @typing.overload + def urlopen( + self, + method: str, + url: str, + redirect: bool = True, + *, + multiplexed: Literal[True], + **kw: typing.Any, + ) -> ResponsePromise: + ... + + def urlopen( self, method: str, url: str, redirect: bool = True, **kw: typing.Any - ) -> BaseHTTPResponse: + ) -> HTTPResponse | ResponsePromise: """ Same as :meth:`urllib3.HTTPConnectionPool.urlopen` with custom cross-host redirect logic and only sends the request-uri @@ -451,6 +649,18 @@ def urlopen( # type: ignore[override] else: response = conn.urlopen(method, u.request_uri, **kw) + if "multiplexed" in kw and kw["multiplexed"]: + if isinstance(response, ResponsePromise): + response.set_parameter("pm_redirect", redirect) + response.set_parameter("pm_url", url) + assert isinstance(response, ResponsePromise) + + return response + + # the established connection is not capable of doing multiplexed request + kw["multiplexed"] = False + + assert isinstance(response, HTTPResponse) redirect_location = redirect and response.get_redirect_location() if not redirect_location: return response @@ -497,7 +707,7 @@ def urlopen( # type: ignore[override] log.info("Redirecting %s -> %s", url, redirect_location) response.drain_conn() - return self.urlopen(method, redirect_location, **kw) + return self.urlopen(method, redirect_location, **kw) # type: ignore[no-any-return] class ProxyManager(PoolManager): @@ -629,9 +839,37 @@ def _set_proxy_headers( headers_.update(headers) return headers_ - def urlopen( # type: ignore[override] - self, method: str, url: str, redirect: bool = True, **kw: typing.Any - ) -> BaseHTTPResponse: + @typing.overload # type: ignore[override] + def urlopen( + self, + method: str, + url: str, + redirect: bool = True, + *, + multiplexed: Literal[False] = ..., + **kw: typing.Any, + ) -> HTTPResponse: + ... + + @typing.overload + def urlopen( + self, + method: str, + url: str, + redirect: bool = True, + *, + multiplexed: Literal[True], + **kw: typing.Any, + ) -> ResponsePromise: + ... + + def urlopen( + self, + method: str, + url: str, + redirect: bool = True, + **kw: typing.Any, + ) -> HTTPResponse | ResponsePromise: "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." u = parse_url(url) if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme): @@ -641,7 +879,7 @@ def urlopen( # type: ignore[override] headers = kw.get("headers", self.headers) kw["headers"] = self._set_proxy_headers(url, headers) - return super().urlopen(method, url, redirect=redirect, **kw) + return super().urlopen(method, url, redirect=redirect, **kw) # type: ignore[no-any-return] def proxy_from_url(url: str, **kw: typing.Any) -> ProxyManager: diff --git a/src/urllib3/response.py b/src/urllib3/response.py index fdee05c604..614e90d581 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -7,9 +7,9 @@ import re import sys import typing +import warnings import zlib from contextlib import contextmanager -from http.client import HTTPMessage as _HttplibHTTPMessage from socket import timeout as SocketTimeout try: @@ -38,8 +38,8 @@ from ._collections import HTTPHeaderDict from ._typing import _TYPE_BODY -from .backend import LowLevelResponse -from .connection import BaseSSLError, HTTPConnection, HTTPException +from .backend import HttpVersion, LowLevelResponse, ResponsePromise +from .connection import BaseSSLError, HTTPConnection from .exceptions import ( DecodeError, HTTPError, @@ -54,6 +54,8 @@ from .util.retry import Retry if typing.TYPE_CHECKING: + from email.message import Message + from typing_extensions import Literal from .connectionpool import HTTPConnectionPool @@ -334,7 +336,7 @@ def __init__( original_response: LowLevelResponse | None = None, pool: HTTPConnectionPool | None = None, connection: HTTPConnection | None = None, - msg: _HttplibHTTPMessage | None = None, + msg: Message | None = None, retries: Retry | None = None, enforce_content_length: bool = True, request_method: str | None = None, @@ -372,6 +374,14 @@ def __init__( self._fp: LowLevelResponse | typing.IO[typing.Any] | None = None self._original_response = original_response self._fp_bytes_read = 0 + + if msg is not None: + warnings.warn( + "Passing msg=.. is deprecated and no-op in urllib3.future and is scheduled to be removed in a future major.", + DeprecationWarning, + stacklevel=2, + ) + self.msg = msg if body and isinstance(body, (str, bytes)): @@ -396,6 +406,16 @@ def __init__( if preload_content and not self._body: self._body = self.read(decode_content=decode_content) + def is_from_promise(self, promise: ResponsePromise) -> bool: + """ + Determine if this response came from given promise. + """ + return ( + self._fp is not None + and hasattr(self._fp, "from_promise") + and self._fp.from_promise == promise + ) + def get_redirect_location(self) -> str | None | Literal[False]: """ Should we redirect and where to? @@ -510,7 +530,10 @@ def release_conn(self) -> None: if not self._pool or not self._connection: return None - self._pool._put_conn(self._connection) + # todo: propose a better way to handle this case graciously + if self._connection._svn is None or self._connection._svn == HttpVersion.h11: + self._pool._put_conn(self._connection) + self._connection = None def drain_conn(self) -> None: @@ -521,7 +544,7 @@ def drain_conn(self) -> None: """ try: self.read() - except (HTTPError, OSError, BaseSSLError, HTTPException): + except (HTTPError, OSError, BaseSSLError): pass @property @@ -634,7 +657,7 @@ def _error_catcher(self) -> typing.Generator[None, None, None]: raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type] - except (HTTPException, OSError) as e: + except OSError as e: # This includes IncompleteRead. raise ProtocolError(f"Connection broken: {e!r}", e) from e diff --git a/src/urllib3/util/retry.py b/src/urllib3/util/retry.py index 7572bfd26a..7191c28ad2 100644 --- a/src/urllib3/util/retry.py +++ b/src/urllib3/util/retry.py @@ -22,7 +22,7 @@ if typing.TYPE_CHECKING: from ..connectionpool import ConnectionPool - from ..response import BaseHTTPResponse + from ..response import HTTPResponse log = logging.getLogger(__name__) @@ -318,7 +318,7 @@ def parse_retry_after(self, retry_after: str) -> float: return seconds - def get_retry_after(self, response: BaseHTTPResponse) -> float | None: + def get_retry_after(self, response: HTTPResponse) -> float | None: """Get the value of Retry-After in seconds.""" retry_after = response.headers.get("Retry-After") @@ -328,7 +328,7 @@ def get_retry_after(self, response: BaseHTTPResponse) -> float | None: return self.parse_retry_after(retry_after) - def sleep_for_retry(self, response: BaseHTTPResponse) -> bool: + def sleep_for_retry(self, response: HTTPResponse) -> bool: retry_after = self.get_retry_after(response) if retry_after: time.sleep(retry_after) @@ -342,7 +342,7 @@ def _sleep_backoff(self) -> None: return time.sleep(backoff) - def sleep(self, response: BaseHTTPResponse | None = None) -> None: + def sleep(self, response: HTTPResponse | None = None) -> None: """Sleep between retry attempts. This method will respect a server's ``Retry-After`` response header @@ -425,7 +425,7 @@ def increment( self, method: str | None = None, url: str | None = None, - response: BaseHTTPResponse | None = None, + response: HTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, @@ -434,7 +434,7 @@ def increment( :param response: A response object, or None, if the server did not return a response. - :type response: :class:`~urllib3.response.BaseHTTPResponse` + :type response: :class:`~urllib3.response.HTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. diff --git a/src/urllib3/util/ssl_.py b/src/urllib3/util/ssl_.py index b20c0ca519..4daa0813ac 100644 --- a/src/urllib3/util/ssl_.py +++ b/src/urllib3/util/ssl_.py @@ -1,13 +1,17 @@ from __future__ import annotations import hmac +import io import os import socket import sys import typing +import warnings from binascii import unhexlify from hashlib import md5, sha1, sha256 +from .._constant import MOZ_INTERMEDIATE_CIPHERS +from ..contrib.imcc import load_cert_chain as _ctx_load_cert_chain from ..exceptions import ProxySchemeUnsupported, SSLError from .url import _BRACELESS_IPV6_ADDRZ_RE, _IPV4_RE @@ -276,8 +280,8 @@ def create_urllib3_context( else: # avoid relying on cpython default cipher list # and instead retrieve OpenSSL own default. This should make - # urllib3.future less seen by basic firewall anti-bot rules. - context.set_ciphers("DEFAULT") + # urllib3.future less flagged by basic firewall anti-bot rules. + context.set_ciphers(MOZ_INTERMEDIATE_CIPHERS) # Setting the default here, as we may have no ssl module on import cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs @@ -353,6 +357,8 @@ def ssl_wrap_socket( ca_cert_data: None | str | bytes = ..., tls_in_tls: Literal[False] = ..., alpn_protocols: list[str] | None = ..., + certdata: str | bytes | None = ..., + keydata: str | bytes | None = ..., ) -> ssl.SSLSocket: ... @@ -373,6 +379,8 @@ def ssl_wrap_socket( ca_cert_data: None | str | bytes = ..., tls_in_tls: bool = ..., alpn_protocols: list[str] | None = ..., + certdata: str | bytes | None = ..., + keydata: str | bytes | None = ..., ) -> ssl.SSLSocket | SSLTransportType: ... @@ -392,6 +400,8 @@ def ssl_wrap_socket( ca_cert_data: None | str | bytes = None, tls_in_tls: bool = False, alpn_protocols: list[str] | None = None, + certdata: str | bytes | None = None, + keydata: str | bytes | None = None, ) -> ssl.SSLSocket | SSLTransportType: """ All arguments except for server_hostname, ssl_context, and ca_cert_dir have @@ -417,6 +427,10 @@ def ssl_wrap_socket( Use SSLTransport to wrap the existing socket. :param alpn_protocols: Manually specify other protocols to be announced during tls handshake. + :param certdata: + Specify an in-memory client intermediary certificate for mTLS. + :param keydata: + Specify an in-memory client intermediary key for mTLS. """ context = ssl_context if context is None: @@ -445,6 +459,15 @@ def ssl_wrap_socket( context.load_cert_chain(certfile, keyfile) else: context.load_cert_chain(certfile, keyfile, key_password) + elif certdata: + try: + _ctx_load_cert_chain(context, certdata, keydata, key_password) + except io.UnsupportedOperation as e: + warnings.warn( + f"""Passing in-memory client/intermediary certificate for mTLS is unsupported on your platform. + Reason: {e}. It will be picked out if you upgrade to a QUIC connection.""", + UserWarning, + ) try: context.set_alpn_protocols(alpn_protocols or ALPN_PROTOCOLS) @@ -496,3 +519,43 @@ def _ssl_wrap_socket_impl( return SSLTransport(sock, ssl_context, server_hostname) return ssl_context.wrap_socket(sock, server_hostname=server_hostname) + + +def is_capable_for_quic( + ctx: ssl.SSLContext | None, ssl_maximum_version: ssl.TLSVersion | int | None +) -> bool: + """ + Quickly uncover if passed parameters for HTTPSConnection does not exclude QUIC. + Some parameters may defacto exclude HTTP/3 over QUIC. + -> TLS 1.3 required + -> One of the three supported ciphers (listed bellow) + """ + quic_disable: bool = False + + if ctx is not None: + if ( + isinstance(ctx.maximum_version, ssl.TLSVersion) + and ctx.maximum_version <= ssl.TLSVersion.TLSv1_2 + ): + quic_disable = True + else: + any_capable_cipher: bool = False + for cipher_dict in ctx.get_ciphers(): + if cipher_dict["name"] in [ + "TLS_AES_128_GCM_SHA256", + "TLS_AES_256_GCM_SHA384", + "TLS_CHACHA20_POLY1305_SHA256", + # Alias-cipher + "CHACHA20-POLY1305-SHA256", + "AES-256-GCM-SHA384", + "AES-128-GCM-SHA256", + ]: + any_capable_cipher = True + break + if not any_capable_cipher: + quic_disable = True + + if ssl_maximum_version and ssl_maximum_version <= ssl.TLSVersion.TLSv1_2: + quic_disable = True + + return not quic_disable diff --git a/src/urllib3/util/ssltransport.py b/src/urllib3/util/ssltransport.py index 14581b18a4..52cc7f2ba9 100644 --- a/src/urllib3/util/ssltransport.py +++ b/src/urllib3/util/ssltransport.py @@ -1,10 +1,10 @@ from __future__ import annotations -import io import socket import ssl import typing +from .._constant import DEFAULT_BLOCKSIZE from ..exceptions import ProxySchemeUnsupported if typing.TYPE_CHECKING: @@ -17,8 +17,6 @@ _WriteBuffer = typing.Union[bytearray, memoryview] _ReturnValue = typing.TypeVar("_ReturnValue") -SSL_BLOCKSIZE = 16384 - class SSLTransport: """ @@ -114,58 +112,6 @@ def send(self, data: bytes, flags: int = 0) -> int: raise ValueError("non-zero flags not allowed in calls to send") return self._ssl_io_loop(self.sslobj.write, data) - def makefile( - self, - mode: str, - buffering: int | None = None, - *, - encoding: str | None = None, - errors: str | None = None, - newline: str | None = None, - ) -> typing.BinaryIO | typing.TextIO | socket.SocketIO: - """ - Python's httpclient uses makefile and buffered io when reading HTTP - messages and we need to support it. - - This is unfortunately a copy and paste of socket.py makefile with small - changes to point to the socket directly. - """ - if not set(mode) <= {"r", "w", "b"}: - raise ValueError(f"invalid mode {mode!r} (only r, w, b allowed)") - - writing = "w" in mode - reading = "r" in mode or not writing - assert reading or writing - binary = "b" in mode - rawmode = "" - if reading: - rawmode += "r" - if writing: - rawmode += "w" - raw = socket.SocketIO(self, rawmode) # type: ignore[arg-type] - self.socket._io_refs += 1 # type: ignore[attr-defined] - if buffering is None: - buffering = -1 - if buffering < 0: - buffering = io.DEFAULT_BUFFER_SIZE - if buffering == 0: - if not binary: - raise ValueError("unbuffered streams must be binary") - return raw - buffer: typing.BinaryIO - if reading and writing: - buffer = io.BufferedRWPair(raw, raw, buffering) # type: ignore[assignment] - elif reading: - buffer = io.BufferedReader(raw, buffering) - else: - assert writing - buffer = io.BufferedWriter(raw, buffering) - if binary: - return buffer - text = io.TextIOWrapper(buffer, encoding, errors, newline) - text.mode = mode # type: ignore[misc] - return text - def unwrap(self) -> None: self._ssl_io_loop(self.sslobj.unwrap) @@ -272,7 +218,7 @@ def _ssl_io_loop( if errno is None: should_loop = False elif errno == ssl.SSL_ERROR_WANT_READ: - buf = self.socket.recv(SSL_BLOCKSIZE) + buf = self.socket.recv(DEFAULT_BLOCKSIZE) if buf: self.incoming.write(buf) else: diff --git a/test/test_connection.py b/test/test_connection.py index 65c317d84f..da4f51950e 100644 --- a/test/test_connection.py +++ b/test/test_connection.py @@ -2,7 +2,6 @@ import socket import typing -from http.client import ResponseNotReady from unittest import mock import pytest @@ -15,7 +14,7 @@ _url_from_connection, _wrap_proxy_error, ) -from urllib3.exceptions import HTTPError, ProxyError, SSLError +from urllib3.exceptions import HTTPError, ProxyError, ResponseNotReady, SSLError from urllib3.util import ssl_ from urllib3.util.ssl_match_hostname import ( CertificateError as ImplementationCertificateError, diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py index 94b8953cab..c7ca45a39a 100644 --- a/test/test_connectionpool.py +++ b/test/test_connectionpool.py @@ -3,7 +3,6 @@ import http.client as httplib import ssl import typing -from http.client import HTTPException from queue import Empty from socket import error as SocketError from ssl import SSLError as BaseSSLError @@ -343,7 +342,7 @@ def _test( # being raised, a retry will be triggered, but that retry will # fail, eventually raising MaxRetryError, not EmptyPoolError # See: https://github.com/urllib3/urllib3/issues/76 - with patch.object(pool, "_make_request", side_effect=HTTPException()): + with patch.object(pool, "_make_request", side_effect=OSError()): with pytest.raises(MaxRetryError): pool.request("GET", "/", retries=1, pool_timeout=SHORT_TIMEOUT) assert pool.pool is not None @@ -579,7 +578,7 @@ def _test(exception: type[BaseException]) -> None: # Run the test case for all the retriable exceptions. _test(TimeoutError) - _test(HTTPException) + _test(OSError) _test(SocketError) _test(ProtocolError) diff --git a/test/test_poolmanager.py b/test/test_poolmanager.py index 6b859acf22..ebe46a660c 100644 --- a/test/test_poolmanager.py +++ b/test/test_poolmanager.py @@ -8,14 +8,10 @@ import pytest from urllib3 import connection_from_url +from urllib3._constant import DEFAULT_BLOCKSIZE from urllib3.connectionpool import HTTPSConnectionPool from urllib3.exceptions import LocationValueError -from urllib3.poolmanager import ( - _DEFAULT_BLOCKSIZE, - PoolKey, - PoolManager, - key_fn_by_scheme, -) +from urllib3.poolmanager import PoolKey, PoolManager, key_fn_by_scheme from urllib3.util import retry, timeout from urllib3.util.url import Url @@ -96,7 +92,7 @@ def test_http_pool_key_fields(self) -> None: "retries": retry.Retry(total=6, connect=2), "block": True, "source_address": "127.0.0.1", - "blocksize": _DEFAULT_BLOCKSIZE + 1, + "blocksize": DEFAULT_BLOCKSIZE + 1, } p = PoolManager() conn_pools = [ @@ -129,7 +125,7 @@ def test_https_pool_key_fields(self) -> None: "cert_reqs": "CERT_REQUIRED", "ca_certs": "/root/path_to_pem", "ssl_version": "SSLv23_METHOD", - "blocksize": _DEFAULT_BLOCKSIZE + 1, + "blocksize": DEFAULT_BLOCKSIZE + 1, } p = PoolManager() conn_pools = [ @@ -378,8 +374,8 @@ def test_pool_manager_no_url_absolute_form(self) -> None: @pytest.mark.parametrize( "input_blocksize,expected_blocksize", [ - (_DEFAULT_BLOCKSIZE, _DEFAULT_BLOCKSIZE), - (None, _DEFAULT_BLOCKSIZE), + (DEFAULT_BLOCKSIZE, DEFAULT_BLOCKSIZE), + (None, DEFAULT_BLOCKSIZE), (8192, 8192), ], ) diff --git a/test/test_response.py b/test/test_response.py index 64cc0eb3ba..41ab63fb2a 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -911,13 +911,12 @@ def test_buggy_incomplete_read(self) -> None: with pytest.raises(ProtocolError) as ctx: resp.read(3) - orig_ex = ctx.value.args[1] - assert isinstance(orig_ex, IncompleteRead) - assert orig_ex.partial == 0 # type: ignore[comparison-overlap] - assert orig_ex.expected == content_length + assert isinstance(ctx.value, IncompleteRead) + assert ctx.value.partial == 0 + assert ctx.value.expected == content_length def test_chunked_head_response(self) -> None: - def mock_sock(amt: int | None) -> tuple[bytes, bool]: + def mock_sock(amt: int | None, stream_id: int | None) -> tuple[bytes, bool]: return b"", True r = LowLevelResponse("HEAD", 200, HttpVersion.h11, "OK", HTTPHeaderDict(), mock_sock) # type: ignore[arg-type] @@ -1014,7 +1013,7 @@ def stream() -> typing.Generator[bytes, None, None]: chunks = list(stream()) idx = 0 - def mock_sock(amt: int | None) -> tuple[bytes, bool]: + def mock_sock(amt: int | None, stream_id: int | None) -> tuple[bytes, bool]: nonlocal chunks, idx if idx >= len(chunks): return b"", True diff --git a/test/test_ssl.py b/test/test_ssl.py index 715439bb4a..bcd8d2e542 100644 --- a/test/test_ssl.py +++ b/test/test_ssl.py @@ -6,6 +6,7 @@ import pytest +from urllib3._constant import MOZ_INTERMEDIATE_CIPHERS from urllib3.exceptions import ProxySchemeUnsupported, SSLError from urllib3.util import ssl_ @@ -136,7 +137,7 @@ def test_create_urllib3_context_default_ciphers( ssl_.create_urllib3_context() - context.set_ciphers.assert_called_once_with("DEFAULT") + context.set_ciphers.assert_called_once_with(MOZ_INTERMEDIATE_CIPHERS) @pytest.mark.parametrize( "kwargs", diff --git a/test/test_ssltransport.py b/test/test_ssltransport.py index cace51db96..9931b67924 100644 --- a/test/test_ssltransport.py +++ b/test/test_ssltransport.py @@ -1,6 +1,5 @@ from __future__ import annotations -import platform import select import socket import ssl @@ -162,20 +161,6 @@ def test_wrap_existing_socket(self) -> None: response = consume_socket(ssock) validate_response(response) - @pytest.mark.timeout(PER_TEST_TIMEOUT) - def test_unbuffered_text_makefile(self) -> None: - self.start_dummy_server() - - sock = socket.create_connection((self.host, self.port)) - with SSLTransport( - sock, self.client_context, server_hostname="localhost" - ) as ssock: - with pytest.raises(ValueError): - ssock.makefile("r", buffering=0) - ssock.send(sample_request()) - response = consume_socket(ssock) - validate_response(response) - @pytest.mark.timeout(PER_TEST_TIMEOUT) def test_unwrap_existing_socket(self) -> None: """ @@ -424,74 +409,6 @@ def test_wrong_sni_hint(self) -> None: proxy_sock, self.client_context, server_hostname="veryverywrong" ) - @pytest.mark.timeout(PER_TEST_TIMEOUT) - @pytest.mark.parametrize("buffering", [None, 0]) - def test_tls_in_tls_makefile_raw_rw_binary(self, buffering: int | None) -> None: - """ - Uses makefile with read, write and binary modes without buffering. - """ - self.start_destination_server() - self.start_proxy_server() - - sock = socket.create_connection( - (self.proxy_server.host, self.proxy_server.port) - ) - with self.client_context.wrap_socket( - sock, server_hostname="localhost" - ) as proxy_sock: - with SSLTransport( - proxy_sock, self.client_context, server_hostname="localhost" - ) as destination_sock: - file = destination_sock.makefile("rwb", buffering) - file.write(sample_request()) # type: ignore[call-overload] - file.flush() - - response = bytearray(65536) - wrote = file.readinto(response) # type: ignore[union-attr] - assert wrote is not None - # Allocated response is bigger than the actual response, we - # rtrim remaining x00 bytes. - str_response = response.decode("utf-8").rstrip("\x00") - validate_response(str_response, binary=False) - file.close() - - @pytest.mark.skipif( - platform.system() == "Windows", - reason="Skipping windows due to text makefile support", - ) - @pytest.mark.timeout(PER_TEST_TIMEOUT) - def test_tls_in_tls_makefile_rw_text(self) -> None: - """ - Creates a separate buffer for reading and writing using text mode and - utf-8 encoding. - """ - self.start_destination_server() - self.start_proxy_server() - - sock = socket.create_connection( - (self.proxy_server.host, self.proxy_server.port) - ) - with self.client_context.wrap_socket( - sock, server_hostname="localhost" - ) as proxy_sock: - with SSLTransport( - proxy_sock, self.client_context, server_hostname="localhost" - ) as destination_sock: - read = destination_sock.makefile("r", encoding="utf-8") - write = destination_sock.makefile("w", encoding="utf-8") - - write.write(sample_request(binary=False)) # type: ignore[arg-type, call-overload] - write.flush() - - response = read.read() - assert isinstance(response, str) - if "\r" not in response: - # Carriage return will be removed when reading as a file on - # some platforms. We add it before the comparison. - assert isinstance(response, str) - response = response.replace("\n", "\r\n") - validate_response(response, binary=False) - @pytest.mark.timeout(PER_TEST_TIMEOUT) def test_tls_in_tls_recv_into_sendall(self) -> None: """ @@ -549,16 +466,6 @@ def test_various_flags_errors(self) -> None: with pytest.raises(ValueError): ssl_transport.send(None, flags=1) # type: ignore[arg-type] - def test_makefile_wrong_mode_error(self) -> None: - server_hostname = "example-domain.com" - sock = mock.Mock() - context = mock.create_autospec(ssl_.SSLContext) - ssl_transport = SSLTransport( - sock, context, server_hostname=server_hostname, suppress_ragged_eofs=False - ) - with pytest.raises(ValueError): - ssl_transport.makefile(mode="x") - def test_wrap_ssl_read_error(self) -> None: server_hostname = "example-domain.com" sock = mock.Mock() diff --git a/test/with_dummyserver/test_connection.py b/test/with_dummyserver/test_connection.py index d7d66b9462..3a846545b3 100644 --- a/test/with_dummyserver/test_connection.py +++ b/test/with_dummyserver/test_connection.py @@ -1,12 +1,12 @@ from __future__ import annotations import typing -from http.client import ResponseNotReady import pytest from dummyserver.testcase import HTTPDummyServerTestCase as server from urllib3 import HTTPConnectionPool +from urllib3.exceptions import ResponseNotReady from urllib3.response import HTTPResponse diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 0ab7d596ef..4df2e8ab15 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -14,7 +14,7 @@ from dummyserver.server import HAS_IPV6_AND_DNS, NoIPv6Warning from dummyserver.testcase import HTTPDummyServerTestCase, SocketDummyServerTestCase -from urllib3 import HTTPConnectionPool, encode_multipart_formdata +from urllib3 import HTTPConnectionPool, ResponsePromise, encode_multipart_formdata from urllib3._collections import HTTPHeaderDict from urllib3._typing import _TYPE_FIELD_VALUE_TUPLE, _TYPE_TIMEOUT from urllib3.connection import _get_default_user_agent @@ -1085,6 +1085,15 @@ def test_user_agent_non_ascii_user_agent( assert "User-Agent" in request_headers assert request_headers["User-Agent"] == "Schönefeld/1.18.0" + def test_fake_multiplexed_connection(self) -> None: + with HTTPConnectionPool(self.host, self.port, maxsize=3) as pool: + promise_a = pool.request("GET", "/headers", multiplexed=True) + assert not isinstance(promise_a, ResponsePromise) + assert pool.num_connections == 1 + promise_b = pool.request("GET", "/headers", multiplexed=True) + assert not isinstance(promise_b, ResponsePromise) + assert pool.num_connections == 1 + class TestRetry(HTTPDummyServerTestCase): def test_max_retry(self) -> None: diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index 87a6cc93ea..9c74386f85 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -11,6 +11,8 @@ LONG_TIMEOUT, SHORT_TIMEOUT, TARPIT_HOST, + notMacOS, + notWindows, requires_network, resolvesLocalhostFQDN, ) @@ -168,6 +170,23 @@ def test_client_intermediate(self) -> None: subject = r.json() assert subject["organizationalUnitName"].startswith("Testing cert") + @notWindows() + @notMacOS() + def test_in_memory_client_intermediate(self) -> None: + with HTTPSConnectionPool( + self.host, + self.port, + key_data=open(os.path.join(self.certs_dir, CLIENT_INTERMEDIATE_KEY)).read(), + cert_data=open( + os.path.join(self.certs_dir, CLIENT_INTERMEDIATE_PEM) + ).read(), + ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), + ) as https_pool: + r = https_pool.request("GET", "/certificate") + subject = r.json() + assert subject["organizationalUnitName"].startswith("Testing cert") + def test_client_no_intermediate(self) -> None: """Check that missing links in certificate chains indeed break @@ -199,6 +218,22 @@ def test_client_key_password(self) -> None: subject = r.json() assert subject["organizationalUnitName"].startswith("Testing cert") + @notWindows() + @notMacOS() + def test_in_memory_client_key_password(self) -> None: + with HTTPSConnectionPool( + self.host, + self.port, + ca_certs=DEFAULT_CA, + key_data=open(os.path.join(self.certs_dir, PASSWORD_CLIENT_KEYFILE)).read(), + cert_data=open(os.path.join(self.certs_dir, CLIENT_CERT)).read(), + key_password="letmein", + ssl_minimum_version=self.tls_version(), + ) as https_pool: + r = https_pool.request("GET", "/certificate") + subject = r.json() + assert subject["organizationalUnitName"].startswith("Testing cert") + def test_client_encrypted_key_requires_password(self) -> None: with HTTPSConnectionPool( self.host, diff --git a/test/with_traefik/test_connection.py b/test/with_traefik/test_connection.py index fe0a9d489a..f41b3107de 100644 --- a/test/with_traefik/test_connection.py +++ b/test/with_traefik/test_connection.py @@ -1,11 +1,10 @@ from __future__ import annotations -from http.client import ResponseNotReady - import pytest from urllib3 import HttpVersion from urllib3.connection import HTTPSConnection +from urllib3.exceptions import ResponseNotReady from . import TraefikTestCase diff --git a/test/with_traefik/test_connection_multiplexed.py b/test/with_traefik/test_connection_multiplexed.py new file mode 100644 index 0000000000..a7f455a19d --- /dev/null +++ b/test/with_traefik/test_connection_multiplexed.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from time import time + +from urllib3.connection import HTTPSConnection + +from . import TraefikTestCase + + +class TestConnectionMultiplexed(TraefikTestCase): + def test_multiplexing_fastest_to_slowest(self) -> None: + conn = HTTPSConnection(self.host, self.https_port, ca_certs=self.ca_authority) + + promises = [] + + for i in range(5): + promises.append(conn.request("GET", f"/delay/{i + 1}")) + promises.append(conn.request("GET", f"/delay/{i + 1}")) + + assert len(promises) == 10 + + before = time() + + for i, expected_wait in zip(range(10), [1, 1, 2, 2, 3, 3, 4, 4, 5, 5]): + r = conn.getresponse() + + assert r.version == 20 + assert r.json()["url"].endswith(f"/delay/{expected_wait}") + + delay = round(time() - before, 2) + + assert expected_wait + 0.5 >= delay + + conn.close() + + def test_multiplexing_slowest_to_fastest(self) -> None: + conn = HTTPSConnection(self.host, self.https_port, ca_certs=self.ca_authority) + + promises = [] + + for i in [5, 4, 3, 2, 1]: + promises.append(conn.request("GET", f"/delay/{i}")) + promises.append(conn.request("GET", f"/delay/{i}")) + + assert len(promises) == 10 + + before = time() + + for expected_wait in [1, 1, 2, 2, 3, 3, 4, 4, 5, 5]: + r = conn.getresponse() + + assert r.version == 20 + assert r.json()["url"].endswith(f"/delay/{expected_wait}") + + delay = round(time() - before, 2) + + assert expected_wait + 0.5 >= delay + + conn.close() + + def test_multiplexing_wait_for_promise(self) -> None: + conn = HTTPSConnection(self.host, self.https_port, ca_certs=self.ca_authority) + + promises = [] + + for i in [5, 4, 3, 2, 1]: + promises.append(conn.request("GET", f"/delay/{i}")) + promises.append(conn.request("GET", f"/delay/{i}")) + + assert len(promises) == 10 + + r = conn.getresponse(promise=promises[2]) # the (first) 4 seconds delay + + assert r.version == 20 + assert r.json()["url"].endswith("/delay/4") + + # empty the promise queue + for i in range(9): + r = conn.getresponse() + assert r.version == 20 + + assert len(conn._promises) == 0 + + def test_multiplexing_upgrade_h3(self) -> None: + conn = HTTPSConnection(self.host, self.https_port, ca_certs=self.ca_authority) + + for i in range(3): + conn.request("GET", "/get") + + for i in range(3): + r = conn.getresponse() + assert r.version == 20 + + for i in range(3): + conn.request("GET", "/get") + + for i in range(3): + r = conn.getresponse() + assert r.version == 30 diff --git a/test/with_traefik/test_connectionpool_multiplexed.py b/test/with_traefik/test_connectionpool_multiplexed.py new file mode 100644 index 0000000000..13f29f2b64 --- /dev/null +++ b/test/with_traefik/test_connectionpool_multiplexed.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from time import time + +from urllib3 import HTTPSConnectionPool, ResponsePromise + +from . import TraefikTestCase + + +class TestConnectionPoolMultiplexed(TraefikTestCase): + def test_multiplexing_fastest_to_slowest(self) -> None: + with HTTPSConnectionPool( + self.host, self.https_port, ca_certs=self.ca_authority + ) as pool: + promises = [] + + for i in range(5): + promise_slow = pool.urlopen("GET", "/delay/3", multiplexed=True) + promise_fast = pool.urlopen("GET", "/delay/1", multiplexed=True) + + assert isinstance(promise_fast, ResponsePromise) + assert isinstance(promise_slow, ResponsePromise) + promises.append(promise_slow) + promises.append(promise_fast) + + assert len(promises) == 10 + + before = time() + + for i in range(5): + response = pool.get_response() + assert response is not None + assert response.status == 200 + assert "/delay/1" in response.json()["url"] + + assert 1.5 >= round(time() - before, 2) + + for i in range(5): + response = pool.get_response() + assert response is not None + assert response.status == 200 + assert "/delay/3" in response.json()["url"] + + assert 3.5 >= round(time() - before, 2) + assert pool.get_response() is None diff --git a/test/with_traefik/test_send_data.py b/test/with_traefik/test_send_data.py index 5b70769bc9..c42f97b979 100644 --- a/test/with_traefik/test_send_data.py +++ b/test/with_traefik/test_send_data.py @@ -66,23 +66,21 @@ def test_h2n3_data(self, method: str, body: bytes | str | BytesIO) -> None: assert resp.status == 200 assert resp.version == (20 if i == 0 else 30) - print(resp.json()["data"]) - payload_seen_by_server: bytes = b64decode(resp.json()["data"][37:]) if isinstance(body, str): assert payload_seen_by_server == body.encode( "utf-8" - ), f"HTTP/{resp.version/10} POST body failure: str" + ), f"HTTP/{resp.version / 10} POST body failure: str" elif isinstance(body, bytes): assert ( payload_seen_by_server == body - ), f"HTTP/{resp.version/10} POST body failure: bytes" + ), f"HTTP/{resp.version / 10} POST body failure: bytes" else: body.seek(0, 0) assert ( payload_seen_by_server == body.read() - ), f"HTTP/{resp.version/10} POST body failure: BytesIO" + ), f"HTTP/{resp.version / 10} POST body failure: BytesIO" @pytest.mark.parametrize( "method", diff --git a/test/with_traefik/test_stream.py b/test/with_traefik/test_stream.py index e12b277112..3834f1a16e 100644 --- a/test/with_traefik/test_stream.py +++ b/test/with_traefik/test_stream.py @@ -46,10 +46,10 @@ def test_h2n3_stream(self, amt: int | None) -> None: assert ( payload_reconstructed is not None - ), f"HTTP/{resp.version/10} stream failure" + ), f"HTTP/{resp.version / 10} stream failure" assert ( "args" in payload_reconstructed - ), f"HTTP/{resp.version/10} stream failure" + ), f"HTTP/{resp.version / 10} stream failure" def test_read_zero(self) -> None: with HTTPSConnectionPool(