Pull upstream changes from #37.

This adjusts the code after the said PR refactored the project
structure.

Reference: #37

Author: BurnzZ

README.rst

@@ -18,8 +18,8 @@ web-poet
    :target: https://codecov.io/gh/scrapinghub/web-poet
    :alt: Coverage report
 
-.. image:: https://readthedocs.org/projects/web-poet/badge/?version=latest
-   :target: https://web-poet.readthedocs.io/en/latest/?badge=latest
+.. image:: https://readthedocs.org/projects/web-poet/badge/?version=stable
+   :target: https://web-poet.readthedocs.io/en/stable/?badge=stable
    :alt: Documentation Status
 
 ``web-poet`` implements Page Object pattern for web scraping.

tests/test_requests.py

@@ -3,15 +3,13 @@
 import pytest
 from web_poet.exceptions import RequestBackendError, HttpResponseError
 from web_poet.page_inputs import (
+    HttpClient,
     HttpRequest,
     HttpResponse,
     HttpRequestBody,
     HttpRequestHeaders
 )
-from web_poet.requests import (
-    HttpClient,
-    request_backend_var,
-)
+from web_poet.requests import request_backend_var
 
 
 @pytest.fixture
@@ -47,7 +45,7 @@ async def test_perform_request_from_httpclient(async_mock):
 async def test_http_client_single_requests(async_mock):
     client = HttpClient(async_mock)
 
-    with mock.patch("web_poet.requests.HttpRequest") as mock_request:
+    with mock.patch("web_poet.page_inputs.client.HttpRequest") as mock_request:
         response = await client.request("url")
         response.url == "url"
 

web_poet/__init__.py

@@ -1,10 +1,8 @@
 from .pages import WebPage, ItemPage, ItemWebPage, Injectable
-from .requests import (
-    request_backend_var,
-    HttpClient,
-)
+from .requests import request_backend_var
 from .page_inputs import (
     Meta,
+    HttpClient,
     HttpRequest,
     HttpResponse,
     HttpRequestHeaders,

web_poet/exceptions/http.py

@@ -6,7 +6,7 @@
 operations.
 """
 
-from web_poet.page_inputs import HttpResponse, HttpRequest
+from web_poet.page_inputs.http import HttpResponse, HttpRequest
 
 
 class HttpError(IOError):

web_poet/page_inputs/__init__.py

@@ -0,0 +1,10 @@
+from .meta import Meta
+from .client import HttpClient
+from .http import (
+    HttpRequest,
+    HttpResponse,
+    HttpRequestHeaders,
+    HttpResponseHeaders,
+    HttpRequestBody,
+    HttpResponseBody,
+)

web_poet/page_inputs/client.py

@@ -0,0 +1,207 @@
+"""This module has a full support for :mod:`asyncio` that enables developers to
+perform asynchronous additional requests inside of Page Objects.
+
+Note that the implementation to fully execute any :class:`~.Request` is not
+handled in this module. With that, the framework using **web-poet** must supply
+the implementation.
+
+You can read more about this in the :ref:`advanced-downloader-impl` documentation.
+"""
+
+import asyncio
+import logging
+from typing import Optional, Dict, List, Union, Callable
+
+from web_poet.requests import request_backend_var, _perform_request
+from web_poet.page_inputs.http import (
+    HttpRequest,
+    HttpRequestHeaders,
+    HttpRequestBody,
+    HttpResponse,
+)
+from web_poet.exceptions import RequestBackendError, HttpResponseError
+from web_poet.utils import as_list
+
+logger = logging.getLogger(__name__)
+
+_StrMapping = Dict[str, str]
+_Headers = Union[_StrMapping, HttpRequestHeaders]
+_Body = Union[bytes, HttpRequestBody]
+_Status = Union[str, int]
+
+
+class HttpClient:
+    """A convenient client to easily execute requests.
+
+    By default, it uses the request implementation assigned in the
+    ``web_poet.request_backend_var`` which is a :mod:`contextvars` instance to
+    download the actual requests. However, it can easily be overridable by
+    providing an optional ``request_downloader`` callable.
+
+    Providing the request implementation by dependency injection would be a good
+    alternative solution when you want to avoid setting up :mod:`contextvars`
+    like ``web_poet.request_backend_var``.
+
+    In any case, this doesn't contain any implementation about how to execute
+    any requests fed into it. When setting that up, make sure that the downloader
+    implementation returns a :class:`~.HttpResponse` instance.
+    """
+
+    def __init__(self, request_downloader: Callable = None):
+        self._request_downloader = request_downloader or _perform_request
+
+    @staticmethod
+    def _handle_status(
+        response: HttpResponse,
+        request: HttpRequest,
+        *,
+        allow_status: List[_Status] = None,
+    ) -> None:
+        allow_status_normalized = list(map(str, as_list(allow_status)))
+        allow_all_status = any(
+            [True for s in allow_status_normalized if "*" == s.strip()]
+        )
+
+        if (
+            allow_all_status
+            or response.status is None  # allows serialized responses from tests
+            or response.status < 400
+            or str(response.status) in allow_status_normalized
+        ):
+            return
+
+        if 400 <= response.status < 500:
+            kind = "Client"
+        elif 500 <= response.status < 600:
+            kind = "Server"
+
+        msg = f"{response.status} {kind} Error for {response.url}"
+        raise HttpResponseError(msg, request=request, response=response)
+
+    async def request(
+        self,
+        url: str,
+        *,
+        method: str = "GET",
+        headers: Optional[_Headers] = None,
+        body: Optional[_Body] = None,
+        allow_status: List[_Status] = None,
+    ) -> HttpResponse:
+        """This is a shortcut for creating a :class:`~.HttpRequest` instance and
+        executing that request.
+
+        A :class:`~.HttpResponse` instance should then be returned for successful
+        responses in the 100-3xx status code range. Otherwise, an exception of
+        type :class:`web_poet.exceptions.http.HttpResponseError` will be raised.
+
+        This behavior can be changed by suppressing the exceptions on select
+        status codes using the ``allow_status`` param:
+            * Passing status code values would not raise the exception when it
+              occurs. This would return the response as-is.
+            * Passing a "*" value would basically allow any status codes.
+
+        .. warning::
+            By convention, the request implementation supplied optionally to
+            :class:`~.HttpClient` should return a :class:`~.HttpResponse` instance.
+            However, the underlying implementation supplied might change that,
+            depending on how the framework using **web-poet** implements it.
+        """
+        headers = headers or {}
+        body = body or b""
+        req = HttpRequest(url=url, method=method, headers=headers, body=body)
+        response = await self.execute(req, allow_status=allow_status)
+        return response
+
+    async def get(
+        self,
+        url: str,
+        *,
+        headers: Optional[_Headers] = None,
+        allow_status: List[_Status] = None,
+    ) -> HttpResponse:
+        """Similar to :meth:`~.HttpClient.request` but peforming a ``GET``
+        request.
+        """
+        return await self.request(
+            url=url,
+            method="GET",
+            headers=headers,
+            allow_status=allow_status,
+        )
+
+    async def post(
+        self,
+        url: str,
+        *,
+        headers: Optional[_Headers] = None,
+        body: Optional[_Body] = None,
+        allow_status: List[_Status] = None,
+    ) -> HttpResponse:
+        """Similar to :meth:`~.HttpClient.request` but performing a ``POST``
+        request.
+        """
+        return await self.request(
+            url=url,
+            method="POST",
+            headers=headers,
+            body=body,
+            allow_status=allow_status,
+        )
+
+    async def execute(
+        self, request: HttpRequest, *, allow_status: List[_Status] = None
+    ) -> HttpResponse:
+        """Accepts a single instance of :class:`~.HttpRequest` and executes it
+        using the request implementation configured in the :class:`~.HttpClient`
+        instance.
+
+        A :class:`~.HttpResponse` instance should then be returned for successful
+        responses in the 100-3xx status code range. Otherwise, an exception of
+        type :class:`web_poet.exceptions.http.HttpResponseError` will be raised.
+
+        This behavior can be changed by suppressing the exceptions on select
+        status codes using the ``allow_status`` param:
+
+            * Passing status code values would not raise the exception when it
+              occurs. This would return the response as-is.
+            * Passing a "*" value would basically allow any status codes.
+        """
+        response = await self._request_downloader(request)
+        self._handle_status(response, request, allow_status=allow_status)
+        return response
+
+    async def batch_execute(
+        self,
+        *requests: HttpRequest,
+        return_exceptions: bool = False,
+        allow_status: List[_Status] = None,
+    ) -> List[Union[HttpResponse, Exception]]:
+        """Similar to :meth:`~.HttpClient.execute` but accepts a collection of
+        :class:`~.HttpRequest` instances that would be batch executed.
+
+        The order of the :class:`~.HttpResponses` would correspond to the order
+        of :class:`~.HttpRequest` passed.
+
+        If any of the :class:`~.HttpRequest` raises an exception upon execution,
+        the exception is raised.
+
+        To prevent this, the actual exception can be returned alongside any
+        successful :class:`~.HttpResponse`. This enables salvaging any usable
+        responses despite any possible failures. This can be done by setting
+        ``True`` to the ``return_exceptions`` parameter.
+
+        Like :meth:`~.HttpClient.execute`, :class:`web_poet.exceptions.http.HttpResponseError`
+        will be raised for responses with status codes in the ``400-5xx`` range.
+        The ``allow_status`` parameter could be used the same way here to prevent
+        these exceptions from being raised.
+
+        You can omit ``allow_status="*"`` if you're passing ``return_exceptions=True``.
+        However, it would be returning :class:`web_poet.exceptions.http.HttpResponseError`
+        instead of :class:`~.HttpResponse`.
+        """
+
+        coroutines = [self.execute(r, allow_status=allow_status) for r in requests]
+        responses = await asyncio.gather(
+            *coroutines, return_exceptions=return_exceptions
+        )
+        return responses

web_poet/page_inputs.py renamed to web_poet/page_inputs/http.py

@@ -14,7 +14,8 @@
 from web_poet.utils import memoizemethod_noargs
 
 T_headers = TypeVar("T_headers", bound="HttpResponseHeaders")
-AnyStrDict = Dict[AnyStr, Union[AnyStr, List[AnyStr], Tuple[AnyStr, ...]]]
+
+_AnyStrDict = Dict[AnyStr, Union[AnyStr, List[AnyStr], Tuple[AnyStr, ...]]]
 
 
 class HttpRequestBody(bytes):
@@ -99,7 +100,7 @@ class HttpResponseHeaders(_HttpHeaders):
 
     @classmethod
     def from_bytes_dict(
-        cls: Type[T_headers], arg: AnyStrDict, encoding: str = "utf-8"
+        cls: Type[T_headers], arg: _AnyStrDict, encoding: str = "utf-8"
     ) -> T_headers:
         """An alternative constructor for instantiation where the header-value
         pairs could be in raw bytes form.
@@ -270,13 +271,3 @@ def _auto_detect_fun(self, body: bytes) -> Optional[str]:
             except UnicodeError:
                 continue
             return resolve_encoding(enc)
-
-
-class Meta(dict):
-    """Container class that could contain any arbitrary data to be passed into
-    a Page Object.
-
-    Note that this is simply a subclass of Python's ``dict``.
-    """
-
-    pass

web_poet/page_inputs/meta.py

@@ -0,0 +1,8 @@
+class Meta(dict):
+    """Container class that could contain any arbitrary data to be passed into
+    a Page Object.
+
+    Note that this is simply a subclass of Python's ``dict``.
+    """
+
+    pass