Merge pull request #7 from thewebscraping/develop

Develop

Author: thewebscraping

tls_requests/__version__.py

@@ -3,5 +3,5 @@
 __url__ = "https://github.com/thewebscraping/tls-requests"
 __author__ = "Tu Pham"
 __author_email__ = "[email protected]"
-__version__ = "1.0.5"
+__version__ = "1.0.6"
 __license__ = "MIT"

tls_requests/client.py

@@ -342,7 +342,7 @@ def _send(
         history = history if isinstance(history, list) else []
         start = start or time.perf_counter()
         config = self.prepare_config(request)
-        response = Response.from_tls_response(self.session.request(config.to_dict()))
+        response = Response.from_tls_response(self.session.request(config.to_dict()), is_byte_response=config.isByteResponse)
         response.request = request
         response.default_encoding = self.encoding
         response.elapsed = datetime.timedelta(seconds=time.perf_counter() - start)

tls_requests/exceptions.py

@@ -8,7 +8,6 @@
 __all__ = [
     "CookieConflictError",
     "HTTPError",
-    "HTTPStatusError",
     "URLError",
     "RemoteProtocolError",
     "ProtocolError",
@@ -38,13 +37,12 @@ class TooManyRedirects(HTTPError):
     """Too Many Redirects."""
 
 
-# Client errors
 class TLSError(HTTPError):
     """TLS Error"""
 
 
-class HTTPStatusError(HTTPError):
-    """HTTP Status Error"""
+class Base64DecodeError(HTTPError):
+    """Base64 Decode Error"""
 
 
 class URLError(HTTPError):

tls_requests/models/libraries.py

@@ -21,7 +21,7 @@
     OS_MACHINE = "amd64"
 
 PATTERN_RE = re.compile(
-    r"[a-zA-Z0-9.-]+%s-%s\.(so|dll|dylib)" % (OS_PLATFORM, OS_MACHINE), re.I
+    r"%s-%s\.(so|dll|dylib)" % (OS_PLATFORM, OS_MACHINE), re.I
 )
 
 
@@ -72,16 +72,17 @@ def fetch_api(cls, version: str = None, retries: int = 3):
                     releases = [
                         Release.from_kwargs(**kwargs) for kwargs in response_json
                     ]
+
+                    if version is not None:
+                        version = "v%s" % version if not str(version).startswith("v") else str(version)
+                        releases = [release for release in releases if version == release.name]
+
                     assets = [
                         asset
                         for release in releases
                         for asset in release.assets
                         if PATTERN_RE.search(asset.browser_download_url)
                     ]
-                    if version is not None:
-                        for asset in assets:
-                            if str(version) == asset.name:
-                                yield asset.browser_download_url
 
                     for asset in assets:
                         yield asset.browser_download_url

tls_requests/models/response.py

@@ -1,9 +1,10 @@
+import binascii
 import codecs
 import datetime
 from email.message import Message
 from typing import Any, Callable, Optional, TypeVar, Union
 
-from tls_requests.exceptions import HTTPError
+from tls_requests.exceptions import Base64DecodeError, HTTPError
 from tls_requests.models.cookies import Cookies
 from tls_requests.models.encoders import StreamEncoder
 from tls_requests.models.headers import Headers
@@ -12,7 +13,7 @@
 from tls_requests.models.tls import TLSResponse
 from tls_requests.settings import CHUNK_SIZE
 from tls_requests.types import CookieTypes, HeaderTypes, ResponseHistory
-from tls_requests.utils import chardet, to_json
+from tls_requests.utils import b64decode, chardet, to_json
 
 __all__ = ["Response"]
 
@@ -231,10 +232,21 @@ async def aclose(self) -> None:
         return self.close()
 
     @classmethod
-    def from_tls_response(cls, response: TLSResponse) -> "Response":
+    def from_tls_response(cls, response: TLSResponse, is_byte_response: bool = False) -> "Response":
+        def _parse_response_body(value: Optional[str]) -> bytes:
+            if value:
+                if is_byte_response:
+                    try:
+                        value = b64decode(value.split(",")[-1])
+                        return value
+                    except (binascii.Error, AssertionError):
+                        raise Base64DecodeError("Couldn't decode the base64 string into bytes.")
+                return value.encode("utf-8")
+            return b""
+
         ret = cls(
             status_code=response.status,
-            body=response.body.encode("utf-8") if response.body else b"",
+            body=_parse_response_body(response.body),
             headers=response.headers,
             cookies=response.cookies,
         )

tls_requests/models/tls.py

@@ -418,6 +418,7 @@ class TLSConfig(_BaseConfig):
     headers: dict[str, str] = field(default_factory=dict)
     insecureSkipVerify: bool = False
     isByteRequest: bool = False
+    isByteResponse: bool = True
     isRotatingProxy: bool = False
     proxyUrl: str = ""
     requestBody: Union[str, bytes, bytearray, None] = None

tls_requests/settings.py

@@ -14,5 +14,5 @@
     "accept": "*/*",
     "connection": "keep-alive",
     "user-agent": f"Python-TLS-Requests/{__version__}",
-    "accept-encoding": "gzip, deflate, br",
+    "accept-encoding": "gzip, deflate, br, zstd",
 }

tls_requests/utils.py

@@ -3,8 +3,7 @@
 import base64
 import importlib
 import logging
-import typing
-from typing import Any, Union
+from typing import Any, AnyStr, Union
 
 FORMAT = "%(levelname)s:%(asctime)s:%(_name)s:%(funcName)s:%(lineno)d >>> %(message)s"
 DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
@@ -74,15 +73,17 @@ def to_str(
     return str(value)
 
 
-def to_base64(value: Union[dict, str, bytes], encoding: str = "utf-8") -> typing.AnyStr:
+def to_base64(value: Union[dict, str, bytes], encoding: str = "utf-8") -> AnyStr:
     return base64.b64encode(to_bytes(value, encoding)).decode(encoding)
 
 
+def b64decode(value: AnyStr) -> bytes:
+    return base64.b64decode(value)
+
+
 def to_json(value: Union[str, bytes], encoding: str = "utf-8", **kwargs) -> dict:
     if isinstance(value, dict):
         return value
-    if isinstance(value, bytes):
-        value = value.decode(encoding, errors="ignore")
     try:
         json_data = jsonlib.loads(value, **kwargs)
         return json_data