Skip to content

Commit

Permalink
Starting to work on ThreadsafeBrowser.request
Browse files Browse the repository at this point in the history
Related to #957
  • Loading branch information
Yomguithereal committed Apr 15, 2024
1 parent 2971c4f commit 244511c
Show file tree
Hide file tree
Showing 3 changed files with 37 additions and 2 deletions.
6 changes: 6 additions & 0 deletions ftest/playwright_request.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
from minet.browser import ThreadsafeBrowser

with ThreadsafeBrowser(headless=False) as browser:
response = browser.request("http://lemonde.fr")

print(response)
5 changes: 3 additions & 2 deletions minet/browser/plawright_shim.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,10 @@ def run_playwright(*args: str) -> int:
env = get_driver_env()
env.setdefault("PLAYWRIGHT_BROWSERS_PATH", get_browsers_path())

driver_executable = compute_driver_executable()
node, pw = compute_driver_executable()

completed_process = subprocess.run(
[str(driver_executable), *args], env=env, stdout=subprocess.DEVNULL
[node, pw, *args], env=env, stdout=subprocess.DEVNULL
)

return completed_process.returncode
Expand Down
28 changes: 28 additions & 0 deletions minet/browser/threadsafe_browser.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,14 @@
from concurrent.futures import Future
from threading import Thread, Event, Lock
from playwright.async_api import async_playwright, Browser, BrowserContext
from urllib3._collections import HTTPHeaderDict

from minet.exceptions import UnknownBrowserError
from minet.web import Response
from minet.browser.plawright_shim import install_browser
from minet.browser.utils import get_browsers_path, get_temp_persistent_context_path
from minet.browser.extensions import get_extension_path, ensure_extension_is_downloaded
from minet.types import Redirection

SUPPORTED_BROWSERS = ("chromium", "firefox")

Expand Down Expand Up @@ -227,3 +230,28 @@ def run_in_browser_or_default_context(
)

return self.__handle_future(future)

async def __request(self, context: BrowserContext, url: str) -> Response:
async with await context.new_page() as page:
emulated_response = await page.goto(url)

assert emulated_response is not None

headers = HTTPHeaderDict()

for header in await emulated_response.headers_array():
headers[header["name"]] = header["value"]

response = Response(
url,
[Redirection(url, status=emulated_response.status)],
headers,
emulated_response.status,
(await page.content()).encode(),
known_encoding="utf-8",
)

return response

def request(self, url: str) -> Response:
return self.run_in_default_context(self.__request, url)

0 comments on commit 244511c

Please sign in to comment.