Merge pull request #26 from mislav/tool-updates

Tool updates

Author: lopuhin

.gitignore

@@ -1,4 +1,3 @@
 .mypy_cache/
 __pycache__/
-extractors/go_readability/go_readability_cli
-extractors/go_domdistiller/go_domdistiller_cli
+venv

Makefile

@@ -0,0 +1,41 @@
+.PHONY: setup
+setup:
+	pip install -r requirements.txt
+# for news_please:
+	python -m nltk.downloader --dir "${VIRTUAL_ENV}"/nltk_data punkt_tab
+	cd extractors/readability_js && npm install --no-audit --no-fund
+	make -C extractors/go_domdistiller
+	make -C extractors/go_readability
+	make -C extractors/go_readability_readeck
+	make -C extractors/go_trafilatura
+
+.PHONY: run-all
+run-all: run-go run-python
+	python extractors/run_readability_js.py
+
+.PHONY: run-go
+run-go: setup
+	python extractors/go_domdistiller.py
+	python extractors/run_go_readability_readeck.py
+	python extractors/run_go_readability.py
+	python extractors/run_go_trafilatura.py
+
+.PHONY: run-python
+run-python: setup
+	python extractors/run_beautifulsoup.py
+# PYTHON2	extractors/run_boilerpipe.py
+# ERRORED	extractors/run_dragnet.py
+	python extractors/run_goose3.py
+	python extractors/run_html_text.py
+	python extractors/run_html2text.py
+	python extractors/run_inscriptis.py
+	python extractors/run_justext.py
+	python extractors/run_readability.py
+	python extractors/run_trafilatura.py
+	python extractors/run_xpath_text.py
+
+.PHONY: run-slow
+# These libraries use machine learning inference, so they can be extremely slow
+run-slow: setup
+	python extractors/run_news_please.py
+	python extractors/run_newspaper.py

README.rst

@@ -6,13 +6,15 @@ extraction for commercial services
 `Zyte Automatic Extraction (ours) <https://www.zyte.com/data-types/news-scraping-api/>`_,
 `Diffbot <https://www.diffbot.com/>`_
 and open-source libraries
-`newspaper3k <https://newspaper.readthedocs.io/en/latest/>`_,
+`newspaper4k <https://github.com/AndyTheFactory/newspaper4k>`_,
 `readability-lxml <https://github.com/buriy/python-readability>`_,
 `dragnet <https://github.com/dragnet-org/dragnet>`_,
 `boilerpipe <https://github.com/misja/python-boilerpipe>`_,
 `html-text <https://github.com/TeamHG-Memex/html-text>`_,
 `trafilatura <https://github.com/adbar/trafilatura>`_,
+`go-trafilatura <https://github.com/markusmobius/go-trafilatura>`_,
 `go-readability <https://github.com/go-shiori/go-readability>`_,
+`readeck/go-readability <https://codeberg.org/readeck/go-readability>`_,
 `Readability.js <https://github.com/mozilla/readability>`_,
 `Go-DomDistiller <https://github.com/markusmobius/go-domdistiller>`_.
 `news-please <https://github.com/fhamborg/news-please>`_.
@@ -47,7 +49,25 @@ Results of the initial evaluation, done in November 2019::
     readability-lxml  0.7.1     0.922 ± 0.014  0.913 ± 0.014  0.931 ± 0.016  0.315 ± 0.035
     xpath-text        4.4.2     0.394 ± 0.020  0.246 ± 0.016  0.992 ± 0.001  0.000 ± 0.000
 
-Result of packages added after original evaluation::
+Results of the latest evaluation with open source libraries added::
+
+                         version    F1             precision      recall         accuracy
+    go-trafilatura       ae7ea06    0.960 ± 0.007  0.940 ± 0.009  0.980 ± 0.006  0.287 ± 0.033
+    trafilatura          2.0.0      0.958 ± 0.006  0.938 ± 0.009  0.978 ± 0.006  0.293 ± 0.033
+    newspaper4k          0.9.3.1    0.949 ± 0.008  0.964 ± 0.008  0.934 ± 0.011  0.326 ± 0.033
+    news_please          1.6.16     0.948 ± 0.008  0.964 ± 0.008  0.933 ± 0.011  0.326 ± 0.034
+    readability_js       0.6.0      0.947 ± 0.005  0.914 ± 0.008  0.982 ± 0.003  0.166 ± 0.028
+    go_readability_fork  fb0fbc5    0.947 ± 0.005  0.914 ± 0.008  0.982 ± 0.003  0.166 ± 0.028
+    go_readability       9f5bf5c    0.934 ± 0.009  0.900 ± 0.011  0.971 ± 0.009  0.188 ± 0.029
+    go_domdistiller      25b8d04    0.927 ± 0.007  0.901 ± 0.010  0.956 ± 0.009  0.061 ± 0.017
+    readability-lxml     0.8.4.1    0.922 ± 0.013  0.913 ± 0.014  0.931 ± 0.015  0.315 ± 0.034
+    goose3               3.1.20     0.896 ± 0.015  0.940 ± 0.013  0.856 ± 0.020  0.232 ± 0.031
+    beautifulsoup        4.13.5     0.860 ± 0.016  0.850 ± 0.016  0.870 ± 0.020  0.006 ± 0.006
+    justext              3.0.2      0.804 ± 0.018  0.858 ± 0.016  0.756 ± 0.027  0.088 ± 0.021
+    inscriptis           2.6.0      0.679 ± 0.015  0.517 ± 0.018  0.992 ± 0.001  0.000 ± 0.000
+    html2text            2025.4.15  0.662 ± 0.015  0.499 ± 0.017  0.983 ± 0.002  0.000 ± 0.000
+
+Earlier results from April 2021::
 
                       version   F1             precision      recall         accuracy
     trafilatura       0.5.1     0.945 ± 0.009  0.925 ± 0.011  0.966 ± 0.009  0.221 ± 0.031
@@ -108,15 +128,17 @@ In addition to benchmarking AutoExtract and Diffbot services, we also benchmark
 open-source libraries that work directly on HTML files without a need for rendering
 or external resources:
 
-- newspaper3k: https://github.com/codelucas/newspaper
+- newspaper4k: https://github.com/AndyTheFactory/newspaper4k
 - readability-lxml: https://github.com/buriy/python-readability
 - dragnet: https://github.com/dragnet-org/dragnet
 - boilerpipe: https://github.com/misja/python-boilerpipe
 - html-text: https://github.com/TeamHG-Memex/html-text -
   this is a baseline which extracts the full text of HTML page
 - trafilatura: https://github.com/adbar/trafilatura contributed by the author
   at https://github.com/scrapinghub/article-extraction-benchmark/pull/4
+- go-trafilatura: https://github.com/markusmobius/go-trafilatura
 - go-readability: https://github.com/go-shiori/go-readability
+- readeck/go-readability: https://codeberg.org/readeck/go-readability/src/branch/main/FORK.md
 - Readability.js: https://github.com/mozilla/readability
 - Go-DomDistiller: https://github.com/markusmobius/go-domdistiller
 - news-please: https://github.com/fhamborg/news-please
@@ -133,21 +155,14 @@ or external resources:
 Output from these libraries is already present in the repo in ``output/*.json`` files.
 They were generated with ``extractors/run_*.py`` files.
 
-All dependencies are in ``requirements.txt``.
-Note that dragnet may fail to install at first try, as
-you need to have ``numpy`` and ``Cython`` installed, and have ``libxml2`` headers
-(``libxml2-dev`` on Ubuntu).
+You can re-generate output JSON files with:
 
-boilerpipe requires a custom installation: use python2, you also need Java
-(e.g. install ``default-jre`` in Ubuntu), install it with
-``pip install -e git+https://github.com/misja/python-boilerpipe.git@ab3694d7bf695b73f0684a028e70aa816d63e6cb#egg=boilerpipe``
+    python3 -m venv ./venv
+    source ./venv/bin/activate
+    make run-all
 
-go-readability requires a custom installation: see README in ``extractors/go_readability``.
-
-Readability.js require a custom installation: install nodejs and install cli tool:
-``npm install -g [email protected]``
-
-Go-DomDistiller requires a custom installation: see README in ``extractors/go_domdistiller``.
+This will install Python dependencies from ``requirements.txt`` into a
+`virtual environment <https://docs.python.org/3/library/venv.html>`_
 
 Evaluation
 ----------

evaluate.py

@@ -22,14 +22,16 @@ def main():
     args = parser.parse_args()
     ground_truth = load_json(Path('ground-truth.json'))
     metrics_by_name = {}
+
+    print(f'{'':<20} {'F1':<13}  {'precision':<13}  {'recall':<13}  {'accuracy':<13}')
     for path in sorted(Path('output').glob('*.json')):
         name = path.stem
         metrics = evaluate(ground_truth, load_json(path), args.n_bootstrap)
         print('{name:<20} '
-              'precision={precision:.3f} ± {precision_std:.3f}  '
-              'recall={recall:.3f} ± {recall_std:.3f}  '
-              'F1={f1:.3f} ± {f1_std:.3f} '
-              'accuracy={accuracy:.3f} ± {accuracy_std:.3f} '
+              '{f1:.3f} ± {f1_std:.3f}  '
+              '{precision:.3f} ± {precision_std:.3f}  '
+              '{recall:.3f} ± {recall_std:.3f}  '
+              '{accuracy:.3f} ± {accuracy_std:.3f}'
               .format(name=name, **metrics))
         metrics_by_name[name] = metrics
 

extractors/go_domdistiller.py

@@ -3,6 +3,7 @@
 import json
 import os
 import subprocess
+import sys
 from pathlib import Path
 from tempfile import mkstemp
 
@@ -11,25 +12,24 @@
 CLI_PATH = Path('extractors/go_domdistiller/go_domdistiller_cli')
 
 
+def normalize(s: str) -> str:
+    # remove all U+00AD (SOFT HYPHEN)
+    return s.replace('\u00ad', '')
+
+
 def main():
     output = {}
     for path in Path('html').glob('*.html.gz'):
         with gzip.open(path, 'rt', encoding='utf8') as f:
             html = f.read()
         item_id = path.stem.split('.')[0]
 
-        # save html to temp file
-        temp_filepath = mkstemp()[1]
-        with open(temp_filepath, 'wt') as fw:
-            fw.write(html)
-
         # get extracted content from go-domdistiller
-        result = subprocess.run([CLI_PATH, temp_filepath], stdout=subprocess.PIPE)
-
-        # destroy temp file
-        os.remove(temp_filepath)
+        result = subprocess.run(CLI_PATH, input=html, text=True, stdout=subprocess.PIPE)
+        if result.returncode != 0:
+            print("failed: ",path,file=sys.stderr)
 
-        output[item_id] = {'articleBody': result.stdout.decode('utf-8')}
+        output[item_id] = {'articleBody': normalize(result.stdout)}
     (Path('output') / 'go_domdistiller.json').write_text(
         json.dumps(output, sort_keys=True, ensure_ascii=False, indent=4),
         encoding='utf8')

extractors/go_domdistiller/.gitignore

@@ -0,0 +1,2 @@
+vendor
+go_domdistiller_cli

extractors/go_domdistiller/Makefile

@@ -0,0 +1,2 @@
+go_domdistiller_cli: cli.go go.mod go.sum
+	go build -o $@ ./cli.go

extractors/go_domdistiller/README.rst

@@ -14,7 +14,7 @@ To use the library I'm wrote a simple cli-module that reads the contents of the
 Installation
 ------------
 
-1. Install golang (I'm used version ``1.15.8``)
+1. Install golang 1.23+
 2. Go to the folder containing this file
 3. Build an executable file:
 

extractors/go_domdistiller/cli.go

@@ -5,26 +5,21 @@ import (
 	"os"
 
 	distiller "github.com/markusmobius/go-domdistiller"
+	"golang.org/x/net/html"
 )
 
 func main() {
-	if len(os.Args) < 2 {
-		panic("Input file not provided in args")
-	}
-	if len(os.Args) > 2 {
-		panic("Args accept only one argument")
+	doc, err := html.Parse(os.Stdin)
+	if err != nil {
+		panic(err)
 	}
-	input := os.Args[1]
 
-	opts := &distiller.Options{
-		ExtractTextOnly: true,
+	article, err := distiller.Apply(doc, &distiller.Options{
 		SkipPagination: true,
-	}
-
-	article, err := distiller.ApplyForFile(input, opts)
+	})
 	if err != nil {
 		panic(err)
 	}
 
-	fmt.Print(article.HTML)
+	fmt.Print(article.Text)
 }

extractors/go_domdistiller/go.mod

@@ -1,5 +1,17 @@
 module cli
 
-go 1.15
+go 1.23
 
-require github.com/markusmobius/go-domdistiller v0.0.0-20201222130639-1c90a88d11c2
+require github.com/markusmobius/go-domdistiller v0.0.0-20240926050704-25b8d046ffb4
+
+require (
+	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect
+	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/rs/zerolog v1.33.0 // indirect
+	golang.org/x/net v0.29.0 // indirect
+	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/text v0.18.0 // indirect
+)