Porting scrape & crawl to newest inferring writer

NOTE: scrape -m is still unsafe
medialab · Dec 15, 2023 · c06e8c6 · c06e8c6
1 parent 8b80fef
commit c06e8c6
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 14 deletions.
diff --git a/ftest/scrapers/title.py b/ftest/scrapers/title.py
@@ -3,8 +3,4 @@
 
 
 def scrape(row: RowWrapper, soup: WonderfulSoup):
-    return {"url": row.url, "title": soup.scrape_one("title")}
-
-
-def titles(row: RowWrapper, soup: WonderfulSoup):
-    yield soup.scrape_one("title")
+    return soup.scrape_one("title")
diff --git a/minet/cli/crawl/crawl.py b/minet/cli/crawl/crawl.py
@@ -85,7 +85,7 @@ def __add_file(self, name: Optional[str], path: str, spider):
 
         if self.format == "csv":
             # TODO: ability to pass fieldnames? from spider?
-            w = casanova.InferringWriter(f, add=["job_id"])
+            w = casanova.InferringWriter(f, prepend=["job_id"])
         elif self.format == "jsonl" or self.format == "ndjson":
             w = ndjson.writer(f)
         else:
@@ -97,7 +97,7 @@ def __unpack_result(self, result: SuccessfulCrawlResult, data):
         job_id = result.job.id
 
         if self.format == "csv":
-            return (data, [job_id])
+            return ([job_id], data)
 
         return ({"job_id": job_id, "data": data},)
 

diff --git a/minet/cli/scrape/scrape.py b/minet/cli/scrape/scrape.py
@@ -194,16 +194,19 @@ def action(cli_args):
 
     if cli_args.format == "csv":
         if isinstance(scraper, FunctionScraper):
-            reader = casanova.reader(cli_args.input, total=cli_args.total)
-
-            # TODO: support for inferring_enricher
-            # TODO: support forwarding cases that will yield None
-            writer = casanova.inferring_writer(
-                cli_args.output, plural_separator=cli_args.plural_separator
+            enricher = casanova.inferring_enricher(
+                cli_args.input,
+                cli_args.output,
+                total=cli_args.total,
+                plural_separator=cli_args.plural_separator,
+                select=cli_args.select,
+                mapping_sample_size=512,
+                buffer_optionals=True,
             )
+            reader = enricher
 
             def writerow(row, item):
-                writer.writerow(item)
+                enricher.writerow(row, item)
 
         else:
             assert scraper.fieldnames is not None
@@ -356,6 +359,8 @@ def payloads() -> Iterator[ScrapeWorkerPayload]:
                     assert result.items is not None
                     items = result.items
 
+                    print(items)
+
                     with writer_lock:
                         for item in items:
                             writerow(original_item.row, item)