From e651190f0b39faa85ff730892c3b690eace3fca7 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Tue, 5 Oct 2021 18:33:17 +0300 Subject: [PATCH] Use response url for original request --- crawlera_fetch/middleware.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crawlera_fetch/middleware.py b/crawlera_fetch/middleware.py index b2a2467..73dac6f 100644 --- a/crawlera_fetch/middleware.py +++ b/crawlera_fetch/middleware.py @@ -260,6 +260,9 @@ def process_response(self, request: Request, response: Response, spider: Spider) url=json_response["url"], body=resp_body, ) + + original_request = original_request.replace(url=json_response["url"]) + return response.replace( cls=respcls, request=original_request,