Crawl source information per item/page

scrapy_webarchive/downloadermiddlewares.py

@@ -56,11 +56,10 @@ def process_request(self, request: Request, spider: Spider):
         if not warc_record:
             self.stats.inc_value("webarchive/response_not_found", spider=spider)
             return Response(url=request.url, status=404)
-
-        request.meta["warc_record_response"] = warc_record
-
+
         # Record found, try to re-create a response from it.
         response = record_transformer.response_for_record(warc_record, request)
+        response.meta["response_origin"] = warc_record
 
         if not response:
             self.stats.inc_value("webarchive/response_not_recognized", spider=spider)

scrapy_webarchive/extensions.py

@@ -61,11 +61,19 @@ def __init__(self, settings: Settings, crawler: Crawler) -> None:
         self.spider_name = crawler.spidercls.name if hasattr(crawler.spidercls, "name") else crawler.spider.name
 
         # Get the store URI and configure the WACZ filename
-        store_uri, self.wacz_fname  = self._retrieve_store_uri_and_wacz_fname()
+        self.store_uri, self.wacz_fname  = self._retrieve_store_uri_and_wacz_fname()
 
-        # Initialize store and writer
-        self.store: FilesStoreProtocol = self._get_store(store_uri)
+        # Initialize store, writer and creator
+        self.store: FilesStoreProtocol = self._get_store(self.store_uri)
         self.writer = WarcFileWriter(collection_name=self.spider_name)
+        self.wacz_creator = WaczFileCreator(
+            store=self.store,
+            warc_fname=self.writer.warc_fname,
+            wacz_fname=self.wacz_fname,
+            collection_name=crawler.spider.name,
+            title=self.settings["SW_WACZ_TITLE"],
+            description=self.settings["SW_WACZ_DESCRIPTION"],
+        )
 
     def _check_configuration_prerequisites(self) -> None:
         """raises NotConfigured if essential settings or middleware configurations are incorrect."""
@@ -143,27 +151,27 @@ def response_downloaded(self, response: Response, request: Request, spider: Spid
         request.meta["WARC-Date"] = get_formatted_dt_string(format=WARC_DT_FORMAT)
 
         # Write response WARC record
-        record = self.writer.write_response(response, request)
+        response_record = self.writer.write_response(response, request)
         self.stats.inc_value("webarchive/exporter/response_written", spider=spider)
         self.stats.inc_value(
-            f"webarchive/exporter/writer_status_count/{record.http_headers.get_statuscode()}", 
+            f"webarchive/exporter/writer_status_count/{response_record.http_headers.get_statuscode()}", 
             spider=spider,
         )
 
         # Write request WARC record
-        self.writer.write_request(request, concurrent_to=record)
+        request_record = self.writer.write_request(request, concurrent_to=response_record)
         self.stats.inc_value("webarchive/exporter/request_written", spider=spider)
 
+        request.meta['exported_warc_response'] = response_record
+        request.meta['exported_warc_request'] = request_record
+        request.meta['exported_wacz_uri'] = self.export_uri
+
     def spider_closed(self, spider: Spider) -> None:
-        wacz_creator = WaczFileCreator(
-            store=self.store,
-            warc_fname=self.writer.warc_fname,
-            wacz_fname=self.wacz_fname,
-            collection_name=spider.name,
-            title=self.settings["SW_WACZ_TITLE"],
-            description=self.settings["SW_WACZ_DESCRIPTION"],
-        )
-        wacz_creator.create()
+        self.wacz_creator.create()
+
+    @property
+    def export_uri(self) -> str:
+        return os.path.join(self.store_uri, self.wacz_creator.wacz_fname)
 
 
 def get_archive_uri_template_dt_variables() -> dict: