From 21c340340717add7f59f581360f8d95c37e72bb1 Mon Sep 17 00:00:00 2001 From: Borys Oliinyk Date: Sat, 15 Jun 2024 17:29:46 +0300 Subject: [PATCH] Update query param extractor method --- crawl/spiders/reverb_com.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crawl/spiders/reverb_com.py b/crawl/spiders/reverb_com.py index 828722c..18a58f5 100644 --- a/crawl/spiders/reverb_com.py +++ b/crawl/spiders/reverb_com.py @@ -52,7 +52,7 @@ class ReverbComSpider(Spider): def start_requests(self) -> Iterator[Request]: for item in get_scraping_links(): if "query" in item["link"]: - query = self._extract_search_param(url=item["link"]) + query = self._extract_query_param(url=item["link"], name="query") payload = self._build_search_request_payload(query=query) callback = self.parse_reverb_search_api else: @@ -135,8 +135,8 @@ def _extract_url_slug(url: str) -> str: return urlparse(url).path.split("/")[-1] @staticmethod - def _extract_search_param(url: str) -> str: - return parse_qs(qs=urlparse(url).query)["query"][0] + def _extract_query_param(url: str, name: str) -> str: + return parse_qs(qs=urlparse(url).query)[name][0] @staticmethod def _build_product_request_payload(