🔖 Release 2.8.904

.github/workflows/integration.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        downstream: [botocore, niquests, requests, boto3, sphinx]
+        downstream: [botocore, niquests, requests, boto3, sphinx, docker]
     runs-on: ubuntu-latest
     timeout-minutes: 30
 
@@ -24,10 +24,16 @@ jobs:
       - name: "Setup Python"
         uses: "actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1"
         with:
-          python-version: "3.x"
+          python-version: "3.11"
 
       - name: "Install dependencies"
         run: python -m pip install --upgrade nox
 
+      - name: "Undo Docker Config: docker-py"
+        if: matrix.downstream == 'docker'
+        run: | 
+          docker logout
+          rm -rf ~/.docker
+
       - name: "Run downstream tests"
         run: nox -s downstream_${{ matrix.downstream }}

CHANGES.rst

@@ -1,3 +1,11 @@
+2.8.904 (2024-07-18)
+====================
+
+- Relaxed h11 constraint around "pending proposal" and coming server event about upgrade.
+  This is made to ensure near perfect compatibility against the legacy urllib3 which is based on http.client.
+- Fixed h11 yielding bytearray instead of bytes in rare circumstances.
+- Added ``docker-py`` in our CI/integration pipeline.
+
 2.8.903 (2024-07-17)
 ====================
 

README.md

@@ -129,7 +129,7 @@ less forgiven in case of bugs than the original urllib3. For good~ish reasons, w
 
 The matter is taken with utmost seriousness and everyone can inspect this package at will.
 
-We regularly test this fork against the most used packages (that depend on urllib3).
+We regularly test this fork against the most used packages (that depend on urllib3, especially those who plunged deep into urllib3 internals).
 
 Finally, rare is someone "fully aware" of their transitive dependencies. And "urllib3" is forced
 into your environments regardless of your preferences.
@@ -151,6 +151,17 @@ Here are some of the reasons (not exhaustive) we choose to work this way:
 - D) Some of our partners started noticing that HTTP/1 started to be disabled by some webservices in favor of HTTP/2+
   So, this fork can unblock them at (almost) zero cost.
 
+**OK... then what do I gain from this?**
+
+- It is faster than its counterpart, we measured gain up to 2X faster in a multithreaded environment using a http2 endpoint.
+- It works well with gevent / does not conflict. We do not use the standard queue class from stdlib as it does not fit http2+ constraints.
+- Leveraging recent protocols like http2 and http3 transparently. Code and behaviors does not change one bit.
+- You do not depend on the standard library to emit http/1 requests, and that is actually a good news. http.client 
+  has numerous known flaws but cannot be fixed as we speak. (e.g. urllib3 is based on http.client)
+- There a ton of other improvement you may leverage, but for that you will need to migrate to Niquests or update your code
+  to enable specific capabilities, like but not limited to: "DNS over QUIC, HTTP" / "Happy Eyeballs" / "Native Asyncio" / "Advanced Multiplexing".
+- Non-blocking IO with concurrent streams/requests. And yes, transparently.
+
 - **Is this funded?**
 
 Yes! We have some funds coming in regularly to ensure its sustainability.

ci/0005-DockerPy-FixBadChunk.patch

@@ -0,0 +1,35 @@
+diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py
+index 3ce127b3..cea350ee 100644
+--- a/tests/unit/api_test.py
++++ b/tests/unit/api_test.py
+@@ -330,6 +330,7 @@ class DockerApiTest(BaseAPIClientTest):
+         content_str = json.dumps(content)
+         content_str = content_str.encode('utf-8')
+         body = io.BytesIO(content_str)
++        body.close = lambda: None  # necessary because get closed after initial preloading.
+
+         # mock a stream interface
+         raw_resp = urllib3.HTTPResponse(body=body)
+@@ -445,7 +446,7 @@ class UnixSocketStreamTest(unittest.TestCase):
+             b'HTTP/1.1 200 OK\r\n'
+             b'Transfer-Encoding: chunked\r\n'
+             b'\r\n'
+-        ) + b'\r\n'.join(lines)
++        ) + b'\r\n'.join(lines) + b'\r\n'  # fix invalid chunked: missing extraneous RC+LF
+
+         with APIClient(
+                 base_url=f"http+unix://{self.socket_file}",
+@@ -460,9 +461,11 @@ class UnixSocketStreamTest(unittest.TestCase):
+                     if i == 4:
+                         raise e
+
+-            assert list(stream) == [
++            # assert assume that sock will yield on each chunk
++            # but not necessarily true.
++            assert b"".join(list(stream)) == b"".join([
+                 str(i).encode() for i in range(50)
+-            ]
++            ])
+
+
+ class TCPSocketStreamTest(unittest.TestCase):

noxfile.py

@@ -282,6 +282,8 @@ def downstream_botocore(session: nox.Session) -> None:
     session.run("python", "scripts/ci/install")
 
     session.cd(root)
+    session.install("setuptools<71")
+
     session.install(".", silent=False)
     session.cd(f"{tmp_dir}/botocore")
 
@@ -401,6 +403,38 @@ def downstream_sphinx(session: nox.Session) -> None:
     )
 
 
+@nox.session()
+def downstream_docker(session: nox.Session) -> None:
+    root = os.getcwd()
+    tmp_dir = session.create_tmp()
+
+    session.cd(tmp_dir)
+    git_clone(session, "https://github.com/docker/docker-py")
+    session.chdir("docker-py")
+
+    for patch in [
+        "0005-DockerPy-FixBadChunk.patch",
+    ]:
+        session.run("git", "apply", f"{root}/ci/{patch}", external=True)
+
+    session.run("git", "rev-parse", "HEAD", external=True)
+    session.install(".[ssh,dev]", silent=False)
+
+    session.cd(root)
+    session.install(".", silent=False)
+    session.cd(f"{tmp_dir}/docker-py")
+
+    session.run("python", "-c", "import urllib3; print(urllib3.__version__)")
+    session.run(
+        "python",
+        "-m",
+        "pytest",
+        "-v",
+        f"--color={'yes' if 'GITHUB_ACTIONS' in os.environ else 'auto'}",
+        *(session.posargs or ("tests/unit",)),
+    )
+
+
 @nox.session()
 def format(session: nox.Session) -> None:
     """Run code formatters."""

src/urllib3/_collections.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import typing
-import warnings
 from collections import OrderedDict
 from enum import Enum, auto
 from functools import lru_cache
@@ -72,7 +71,9 @@ class RecentlyUsedContainer(typing.Generic[_KT, _VT], typing.MutableMapping[_KT,
     """
     Provides a thread-safe dict-like container which maintains up to
     ``maxsize`` keys while throwing away the least-recently-used keys beyond
-    ``maxsize``.
+    ``maxsize``. Caution: RecentlyUsedContainer is deprecated and scheduled for
+    removal in a next major of urllib3.future. It has been replaced by a more
+    suitable implementation in ``urllib3.util.traffic_police``.
 
     :param maxsize:
         Maximum number of recent elements to retain.
@@ -98,13 +99,6 @@ def __init__(
         self._container = OrderedDict()
         self.lock = RLock()
 
-        warnings.warn(
-            "RecentlyUsedContainer is deprecated and scheduled for removal in urllib3.future v3. "
-            "It has been replaced by a more suitable implementation in urllib3.util.traffic_police.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-
     def __getitem__(self, key: _KT) -> _VT:
         # Re-insert the item, moving it to the end of the eviction line.
         with self.lock:

src/urllib3/_version.py

@@ -1,4 +1,4 @@
 # This file is protected via CODEOWNERS
 from __future__ import annotations
 
-__version__ = "2.8.903"
+__version__ = "2.8.904"

src/urllib3/contrib/hface/protocols/http1/_h11.py

@@ -16,9 +16,11 @@
 
 from __future__ import annotations
 
+import warnings
 from functools import lru_cache
 
 import h11
+from h11._state import _SWITCH_UPGRADE, ConnectionState
 
 from ..._stream_matrix import StreamMatrix
 from ..._typing import HeadersType
@@ -100,11 +102,35 @@ def headers_from_response(
     ] + response.headers.raw_items()
 
 
+class RelaxConnectionState(ConnectionState):
+    def process_event(  # type: ignore[no-untyped-def]
+        self,
+        role,
+        event_type,
+        server_switch_event=None,
+    ) -> None:
+        if server_switch_event is not None:
+            if server_switch_event not in self.pending_switch_proposals:
+                if server_switch_event is _SWITCH_UPGRADE:
+                    warnings.warn(
+                        f"Received server {server_switch_event} event without a pending proposal. "
+                        "This will raise an exception in a future version. It is temporarily relaxed to match the "
+                        "legacy http.client standard library.",
+                        DeprecationWarning,
+                        stacklevel=2,
+                    )
+                    self.pending_switch_proposals.add(_SWITCH_UPGRADE)
+
+        return super().process_event(role, event_type, server_switch_event)
+
+
 class HTTP1ProtocolHyperImpl(HTTP1Protocol):
     implementation: str = "h11"
 
     def __init__(self) -> None:
         self._connection: h11.Connection = h11.Connection(h11.CLIENT)
+        self._connection._cstate = RelaxConnectionState()
+
         self._data_buffer: list[bytes] = []
         self._events: StreamMatrix = StreamMatrix()
         self._terminated: bool = False
@@ -255,7 +281,7 @@ def _headers_from_h11_response(
         )
 
     def _data_from_h11(self, h11_event: h11.Data) -> Event:
-        return DataReceived(self._current_stream_id, h11_event.data)
+        return DataReceived(self._current_stream_id, bytes(h11_event.data))
 
     def _connection_terminated(
         self, error_code: int = 0, message: str | None = None