add workaround for Twisted when receiving absolute-form URI in HTTP r…

…equest (#23)

pyproject.toml

@@ -38,7 +38,7 @@ dev = [
     "websocket-client>=1.7.0",
     "coverage[toml]>=5.0.0",
     "coveralls>=3.3",
-    "twisted>=24",
+    "localstack-twisted",
     "ruff==0.1.0"
 ]
 docs = [

rolo/serving/twisted.py

@@ -10,7 +10,7 @@
 from twisted.internet.protocol import Protocol
 from twisted.protocols.policies import ProtocolWrapper
 from twisted.python.components import proxyForInterface
-from twisted.web.http import HTTPChannel, _GenericHTTPChannelProtocol
+from twisted.web.http import HTTPChannel, _GenericHTTPChannelProtocol, urlparse
 from twisted.web.http_headers import Headers as TwistedHeaders
 from twisted.web.resource import IResource
 from twisted.web.server import NOT_DONE_YET, Request, Site
@@ -63,6 +63,20 @@ def update_wsgi_environment(environ: "WSGIEnvironment", request: TwistedRequest)
     # this is needed for streaming requests
     environ["wsgi.input_terminated"] = True
 
+    if not request.path.startswith(b"/"):
+        # TODO: this is a bug in Twisted: when the HTTP request contains a full absolute-form URI (when a request is
+        #  proxied) instead of a relative path, the `PATH_INFO` is wrong, as Twisted will use the full URI as the path.
+        #  `twisted.web.wsgi` will even replace the first char with a slash, leading to something looking like
+        #  '/ttp://sns.eu-central-1.amazonaws.com/'
+        # See RFC7230: https://tools.ietf.org/html/rfc7230#section-5.3.2
+        # > When making a request to a proxy, other than a CONNECT or server-wide OPTIONS request (as detailed below),
+        # > a client MUST send the target URI in absolute-form as the request-target.... An example absolute-form
+        # > of request-line would be:
+        # > GET http://www.example.org/pub/WWW/TheProject.html HTTP/1.1
+        #
+        # we need to fix it upstream, but this is a global workaround for now
+        environ["PATH_INFO"] = urlparse(request.path).path.decode("utf-8")
+
     # create RAW_URI and REQUEST_URI
     environ["REQUEST_URI"] = request.uri.decode("utf-8")
     environ["RAW_URI"] = request.uri.decode("utf-8")
@@ -164,7 +178,6 @@ def protocol_factory():
     def headerReceived(self, line):
         if not super().headerReceived(line):
             return False
-
         # remember casing of headers for requests
         header, data = line.split(b":", 1)
         request: TwistedRequestAdapter = self.requests[-1]

tests/serving/test_twisted.py

@@ -1,4 +1,6 @@
+import http.client
 import io
+import json
 
 import requests
 
@@ -23,3 +25,30 @@ def hello(request: Request):
     response = requests.post(server.url + "/hello", io.BytesIO(b"0" * 100001))
 
     assert response.status_code == 200
+
+
+def test_full_absolute_form_uri(serve_twisted_gateway):
+    router = Router(handler_dispatcher())
+
+    @route("/hello", methods=["GET"])
+    def hello(request: Request):
+        return {"path": request.path, "raw_uri": request.environ["RAW_URI"]}
+
+    router.add(hello)
+
+    gateway = Gateway(request_handlers=[RouterHandler(router, True)])
+    server = serve_twisted_gateway(gateway)
+    host = server.url
+
+    conn = http.client.HTTPConnection(host="127.0.0.1", port=server.port)
+
+    # This is what is sent:
+    # send: b'GET http://localhost:<port>/hello HTTP/1.1\r\nHost: localhost:<port>\r\nAccept-Encoding: identity\r\n\r\n'
+    # note the full URI in the HTTP request
+    conn.request("GET", url=f"{host}/hello")
+    response = conn.getresponse()
+
+    assert response.status == 200
+    response_data = json.loads(response.read())
+    assert response_data["path"] == "/hello"
+    assert response_data["raw_uri"].startswith("http")

tests/test_pydantic.py

@@ -8,6 +8,8 @@
 from rolo.routing import handler as routing_handler
 from rolo.routing import handler_dispatcher
 
+pydantic_version = pydantic.version.version_short()
+
 
 class MyItem(pydantic.BaseModel):
     name: str
@@ -67,7 +69,7 @@ def handler(_request: Request, item_id: int, item: MyItem) -> dict:
                 "msg": "Invalid JSON: EOF while parsing a value at line 1 column 0",
                 "ctx": {"error": "EOF while parsing a value at line 1 column 0"},
                 "input": "",
-                "url": "https://errors.pydantic.dev/2.8/v/json_invalid",
+                "url": f"https://errors.pydantic.dev/{pydantic_version}/v/json_invalid",
             }
         ]
 
@@ -126,7 +128,7 @@ def handler(_request: Request, item_id: int, item: MyItem) -> str:
                 "loc": ["price"],
                 "msg": "Field required",
                 "input": {"name": "rolo"},
-                "url": "https://errors.pydantic.dev/2.8/v/missing",
+                "url": f"https://errors.pydantic.dev/{pydantic_version}/v/missing",
             }
         ]