Skip to content

Commit

Permalink
Adding ThreadsafeBrowser.request timeout
Browse files Browse the repository at this point in the history
Related to #963
  • Loading branch information
Yomguithereal committed Apr 16, 2024
1 parent 3dfdac2 commit 3cef9d2
Show file tree
Hide file tree
Showing 4 changed files with 26 additions and 130 deletions.
4 changes: 1 addition & 3 deletions ftest/playwright_request.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
from minet.browser import ThreadsafeBrowser

with ThreadsafeBrowser(headless=False, adblock=True) as browser:
response = browser.request(
"https://github.com/emsojemgrjgr", raise_on_statuses=(404,)
)
response = browser.request("https://lemonde.fr", raise_on_statuses=(404,))

print(response)
print(response.stack)
Expand Down
121 changes: 0 additions & 121 deletions ftest/pyppeteer.py

This file was deleted.

19 changes: 14 additions & 5 deletions minet/browser/threadsafe_browser.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
Literal,
Container,
)
from minet.types import Concatenate, ParamSpec
from minet.types import Concatenate, ParamSpec, AnyTimeout

import os
import asyncio
Expand All @@ -30,7 +30,7 @@
BrowserYetUnimplementedError,
InvalidStatusError,
)
from minet.web import Response
from minet.web import Response, coerce_timeout_to_milliseconds
from minet.browser.plawright_shim import install_browser
from minet.browser.utils import (
get_browsers_path,
Expand Down Expand Up @@ -260,10 +260,16 @@ async def __request(
context: BrowserContext,
url: str,
raise_on_statuses: Optional[Container[int]] = None,
timeout: Optional[AnyTimeout] = None,
) -> Response:
async with await context.new_page() as page:
try:
emulated_response = await page.goto(url)
actual_timeout = None

if timeout is not None:
actual_timeout = coerce_timeout_to_milliseconds(timeout)

emulated_response = await page.goto(url, timeout=actual_timeout)
except (PlaywrightError, PlaywrightTimeoutError) as e:
error = convert_playwright_error(e)

Expand Down Expand Up @@ -328,8 +334,11 @@ async def __request(
return response

def request(
self, url: str, raise_on_statuses: Optional[Container[int]] = None
self,
url: str,
raise_on_statuses: Optional[Container[int]] = None,
timeout: Optional[AnyTimeout] = None,
) -> Response:
return self.run_in_default_context(
self.__request, url, raise_on_statuses=raise_on_statuses
self.__request, url, raise_on_statuses=raise_on_statuses, timeout=timeout
)
12 changes: 11 additions & 1 deletion minet/web.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ def create_pool_manager(
DEFAULT_POOL_MANAGER = create_pool_manager()


def timeout_to_final_time(timeout: AnyTimeout) -> float:
def coerce_timeout_to_seconds(timeout: AnyTimeout) -> float:
seconds: float

if isinstance(timeout, urllib3.Timeout):
Expand All @@ -179,6 +179,16 @@ def timeout_to_final_time(timeout: AnyTimeout) -> float:
else:
seconds = timeout

return seconds


def coerce_timeout_to_milliseconds(timeout: AnyTimeout) -> float:
return coerce_timeout_to_seconds(timeout) * 1000


def timeout_to_final_time(timeout: AnyTimeout) -> float:
seconds = coerce_timeout_to_seconds(timeout)

# Some epsilon so sockets can timeout themselves properly
seconds += 0.01

Expand Down

0 comments on commit 3cef9d2

Please sign in to comment.