Merge pull request #41 from thewebscraping/feat/rotator

Feature: Smart Rotators & Robust Library Management

Author: thewebscraping

.gitignore

@@ -160,4 +160,4 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
-tls_requests/bin/*xgo*
+tls_requests/bin/*

.pre-commit-config.yaml

@@ -1,12 +1,9 @@
 exclude: '^docs.sh/|scripts/'
 default_stages: [pre-commit]
 
-default_language_version:
-  python: python3.10
-
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v6.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -20,14 +17,21 @@ repos:
       - id: check-docstring-first
       - id: detect-private-key
 
+  # run the autoflake.
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.3.1
+    hooks:
+      - id: autoflake
+        args: [--remove-all-unused-imports, --in-place, --ignore-init-module-imports]
+
   # run the isort.
   - repo: https://github.com/PyCQA/isort
-    rev: 5.13.2
+    rev: 6.1.0
     hooks:
       - id: isort
 
   # run the flake8.
   - repo: https://github.com/PyCQA/flake8
-    rev: 7.0.0
+    rev: 7.3.0
     hooks:
       - id: flake8

Makefile

@@ -2,10 +2,12 @@
 init-actions:
 	python -m pip install --upgrade pip
 	python -m pip install -r requirements-dev.txt
+	python -m autoflake --in-place --remove-all-unused-imports --ignore-init-module-imports .
 	python -m black tls_requests
 	python -m isort tls_requests
 	python -m flake8 tls_requests
 
+
 test:
 	tox -p
 	rm -rf *.egg-info

README.md

@@ -44,6 +44,31 @@ Start using TLS Requests with just a few lines of code:
 200
 ```
 
+Basic automatically rotates:
+
+```pycon
+>>> import tls_requests
+>>> proxy_list = [
+    "http://user1:[email protected]:8080",
+    "http://user2:[email protected]:8081",
+    "socks5://proxy.example.com:8082",
+    "proxy.example.com:8083",  #  (defaults to http)
+    "http://user:[email protected]:8084|1.0|US",  #  http://user:pass@host:port|weight|region
+]
+>>> r = tls_requests.get(
+    "https://httpbin.org/get",
+    proxy=proxy,
+    headers=tls_requests.HeaderRotator(),
+    tls_identifier=tls_requests.TLSIdentifierRotator()
+)
+>>> r
+<Response [200 OK]>
+>>> r.status_code
+200
+>>> tls_requests.HeaderRotator(strategy = "round_robin")  # strategy: Literal["round_robin", "random", "weighted"]
+>>> tls_requests.Proxy("http://user1:[email protected]:8080", weight=0.1)  # default weight: 1.0
+```
+
 **Introduction**
 ----------------
 

docs/advanced/rotators.md

@@ -0,0 +1,145 @@
+# Using Rotators
+
+The `tls_requests` library is designed to be smart out of the box. By default, it automatically rotates through realistic headers and client identifiers to make your requests appear authentic and avoid detection.
+
+This guide explains how these default rotators work and how you can customize or disable them.
+
+* * *
+
+### Header Rotator
+
+**Default Behavior: Automatic Rotation**
+
+When you initialize a `Client` without specifying the `headers` parameter, it will **automatically rotate** through a built-in collection of header templates that mimic popular browsers like Chrome, Firefox, and Safari across different operating systems.
+
+```python
+import tls_requests
+
+# No extra configuration needed!
+# This client will automatically use a different, realistic header set for each request.
+with tls_requests.Client(headers=tls_requests.HeaderRotator()) as client:
+    # Request 1 might have Chrome headers
+    res1 = client.get("https://httpbin.org/headers")
+    print(f"Request 1 UA: {res1.json()['headers']['User-Agent']}")
+
+    # Request 2 might have Firefox headers
+    res2 = client.get("https://httpbin.org/headers")
+    print(f"Request 2 UA: {res2.json()['headers']['User-Agent']}")
+```
+
+**How to Override the Default Behavior:**
+
+-   **To rotate through your own list of headers**, pass a `list` of `dict`s:
+    ```python
+    my_headers = [{"User-Agent": "MyBot/1.0"}, {"User-Agent": "MyBot/2.0"}]
+    client = tls_requests.Client(headers=my_headers)
+    ```
+
+-   **To use a single, static set of headers (no rotation)**, pass a single `dict`:
+    ```python
+    static_headers = {"User-Agent": "Always-The-Same-Bot/1.0"}
+    client = tls_requests.Client(headers=static_headers)
+    ```
+
+-   **To completely disable default headers**, pass `None`:
+    ```python
+    # This client will not add any default headers (like User-Agent).
+    client = tls_requests.Client(headers=None)
+    ```
+
+* * *
+
+### TLS Client Identifier Rotator
+
+**Default Behavior: Automatic Rotation**
+
+Similar to headers, the `Client` **defaults to rotating** through all supported client identifier profiles (e.g., `chrome_120`, `firefox_120`, `safari_16_0`, etc.). This changes your TLS fingerprint with every request, an advanced technique to evade sophisticated anti-bot systems.
+
+```python
+import tls_requests
+
+# This client automatically changes its TLS fingerprint for each request.
+with tls_requests.Client(client_identifier=tls_requests.TLSIdentifierRotator()) as client:
+    # These two requests will have different TLS profiles.
+    res1 = client.get("https://tls.browserleaks.com/json")
+    res2 = client.get("https://tls.browserleaks.com/json")
+```
+
+**How to Override the Default Behavior:**
+
+-   **To rotate through a specific list of identifiers**, pass a `list` of strings:
+    ```python
+    my_identifiers = ["chrome_120", "safari_16_0"]
+    client = tls_requests.Client(client_identifier=my_identifiers)
+    ```
+
+-   **To use a single, static identifier**, pass a string:
+    ```python
+    client = tls_requests.Client(client_identifier="chrome_120")
+    ```
+-   **To disable rotation and use the library's single default identifier**, pass `None`:
+    ```python
+    client = tls_requests.Client(client_identifier=None)
+    ```
+
+* * *
+
+### Proxy Rotator
+
+Unlike headers and client identifiers, proxy rotation is **not enabled by default**, as the library cannot provide a list of free proxies. You must provide your own list to enable this feature.
+
+To enable proxy rotation, pass a list of proxy strings to the `proxy` parameter. The library will automatically use a `weighted` strategy, prioritizing proxies that perform well.
+
+```python
+import tls_requests
+
+proxy_list = [
+    "http://user1:[email protected]:8080",
+    "http://user2:[email protected]:8081",
+    "socks5://proxy.example.com:8082",
+    "proxy.example.com:8083",  #  (defaults to http)
+    "http://user:[email protected]:8084|1.0|US",  #  http://user:pass@host:port|weight|region
+]
+
+# Provide a list to enable proxy rotation.
+with tls_requests.Client(proxy=proxy_list) as client:
+    response = client.get("https://httpbin.org/get")
+```
+
+For more control, you can create a `ProxyRotator` instance with a specific strategy:
+
+```python
+from tls_requests.models.rotators import ProxyRotator
+
+rotator = ProxyRotator.from_file(proxy_list, strategy="round_robin")
+
+with tls_requests.Client(proxy=rotator) as client:
+    response = client.get("https://httpbin.org/get")
+```
+
+> **Note:** The `Client` automatically provides performance feedback (success/failure, latency) to the `ProxyRotator`, making the `weighted` strategy highly effective.
+
+* * *
+
+### Asynchronous Support
+
+All rotator features, including the smart defaults, work identically with `AsyncClient`.
+
+```python
+import tls_requests
+import asyncio
+
+async def main():
+    # This async client automatically uses default header and identifier rotation.
+    async with tls_requests.AsyncClient(
+            headers=tls_requests.HeaderRotator(),
+            client_identifier=tls_requests.TLSIdentifierRotator()
+    ) as client:
+        tasks = [client.get("https://httpbin.org/get") for _ in range(2)]
+        responses = await asyncio.gather(*tasks)
+
+        for i, r in enumerate(responses):
+            print(f"Async Request {i+1} status: {r.status_code}")
+
+asyncio.run(main())
+```

mkdocs.yml

@@ -30,6 +30,7 @@ nav:
         - Authentication: 'advanced/authentication.md'
         - Hooks: 'advanced/hooks.md'
         - Proxies: 'advanced/proxies.md'
+        - Rotators: 'advanced/rotators.md'
     - TLS Client:
         - Install: 'tls/install.md'
         - Wrapper TLS Client: 'tls/index.md'

pyproject.toml

@@ -4,3 +4,47 @@ build-backend = 'setuptools.build_meta'
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
+
+[tool.black]
+line-length = 120
+target-version = ['py38', 'py39', 'py310', 'py311', 'py312']
+unstable = true
+exclude = '''
+/(
+    \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+)/
+'''
+
+
+[tool.flake8]
+max-line-length = 120
+max-complexity = 10
+extend-ignore = [
+    "E203",  # Whitespace before ':', which black handles differently than flake8.
+    "W503",  # Line break before binary operator, black's preferred style.
+]
+
+# Comma-separated list of directories to exclude from linting.
+exclude = [
+    ".git",
+    "__pycache__",
+    "docs/source/conf.py",
+    "old",
+    "build",
+    "dist",
+    ".venv",
+]
+
+# Per-file ignores are very useful for specific cases.
+# For example, __init__.py files often have unused imports on purpose.
+per-file-ignores = [
+    "__init__.py:F401", # Ignore "unused import" in __init__.py files
+]

requirements-dev.txt

@@ -17,6 +17,7 @@ black==24.3.0
 coverage[toml]==7.6.1
 isort==5.13.2
 flake8==7.1.1
+autoflake==2.3.1
 mypy==1.11.2
 pytest==8.3.3
 pytest-asyncio==0.24.0

requirements.txt

@@ -1,5 +1,3 @@
 # Base
 chardet~=5.2.0
-requests~=2.32.3
-tqdm~=4.67.1
 idna~=3.10

setup.cfg

@@ -6,8 +6,6 @@ license_file = LICENSE
 python_requires = >=3.8
 install_requires =
     chardet ~= 5.2.0
-    requests ~= 2.32.3
-    tqdm ~= 4.67.1
     idna ~= 3.10
 classifiers =
     Development Status :: 5 - Production/Stable