From 1a30af12ac5bb689175bb957e94e9f27731b6b21 Mon Sep 17 00:00:00 2001 From: QuaKe8782 Date: Mon, 8 Feb 2021 22:40:59 +0000 Subject: [PATCH] Add rate limit handler --- pypartpicker/scraper.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pypartpicker/scraper.py b/pypartpicker/scraper.py index 6ad480c..75f1925 100644 --- a/pypartpicker/scraper.py +++ b/pypartpicker/scraper.py @@ -57,6 +57,8 @@ def __init__(self, **kwargs): self.rating = kwargs.get("rating") self.content = kwargs.get("content") +class Verification(Exception): + pass class Scraper: @@ -67,20 +69,17 @@ def __init__(self, **kwargs): self.headers = headers_dict - - - def make_soup(self, url) -> BeautifulSoup: # sends a request to the URL page = requests.get(url, headers=self.headers) # gets the HTML code for the website and parses it using Python's built in HTML parser soup = BeautifulSoup(page.content, 'html.parser') + if "Verification" in soup.find(class_="pageTitle").get_text(): + raise Verification("You are being rate limited by PCPartPicker!") # returns the HTML return soup - - def fetch_list(self, list_url) -> PCPPList: # checks if its a pcpartpicker list and raises an exception if its not or if the list is empty @@ -135,8 +134,6 @@ def fetch_list(self, list_url) -> PCPPList: return PCPPList(parts=parts, wattage=wattage, total=total_cost, url=list_url, compatibility=compatibilitynotes) - - def part_search(self, search_term, **kwargs) -> Part: search_term = search_term.replace(' ', '+') @@ -227,8 +224,6 @@ def part_search(self, search_term, **kwargs) -> Part: return parts[:kwargs.get("limit", 20)] - - def fetch_product(self, part_url) -> Product: # checks if the URL is invalid