build: Bump Azure PDF Parser

jesse-c · jesse-c · commit edb457965100 · 2024-07-23T14:49:22.000+01:00
Added an explicit dependency on lxml. This solves the `ImportError` seen:

```
______________________________ ERROR collecting src/html_parser/test/test_readability_parser.py ______________________________
ImportError while importing test module '/Users/jesse/src/github.com/climatepolicyradar/navigator-document-parser/src/html_parser/test/test_readability_parser.py'.
Hint: make sure your test modules/packages have valid Python names.
Traceback:
../../../../Library/Caches/pypoetry/virtualenvs/navigator-document-parser-vnqSsywO-py3.12/lib/python3.12/site-packages/_pytest/python.py:617: in _importtestmodule
    mod = import_path(self.path, mode=importmode, root=self.config.rootpath)
../../../../Library/Caches/pypoetry/virtualenvs/navigator-document-parser-vnqSsywO-py3.12/lib/python3.12/site-packages/_pytest/pathlib.py:567: in import_path
    importlib.import_module(module_name)
/opt/homebrew/Cellar/python@3.12/3.12.4/Frameworks/Python.framework/Versions/3.12/lib/python3.12/importlib/__init__.py:90: in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
&lt;frozen importlib._bootstrap&gt;:1387: in _gcd_import
    ???
&lt;frozen importlib._bootstrap&gt;:1360: in _find_and_load
    ???
&lt;frozen importlib._bootstrap&gt;:1331: in _find_and_load_unlocked
    ???
&lt;frozen importlib._bootstrap&gt;:935: in _load_unlocked
    ???
../../../../Library/Caches/pypoetry/virtualenvs/navigator-document-parser-vnqSsywO-py3.12/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:186: in exec_module
    exec(co, module.__dict__)
src/html_parser/test/test_readability_parser.py:5: in &lt;module&gt;
    from src.html_parser.readability import ReadabilityParser
src/html_parser/readability.py:13: in &lt;module&gt;
    from readability import Document
../../../../Library/Caches/pypoetry/virtualenvs/navigator-document-parser-vnqSsywO-py3.12/lib/python3.12/site-packages/readability/__init__.py:3: in &lt;module&gt;
    from .readability import Document
../../../../Library/Caches/pypoetry/virtualenvs/navigator-document-parser-vnqSsywO-py3.12/lib/python3.12/site-packages/readability/readability.py:11: in &lt;module&gt;
    from .cleaners import clean_attributes
../../../../Library/Caches/pypoetry/virtualenvs/navigator-document-parser-vnqSsywO-py3.12/lib/python3.12/site-packages/readability/cleaners.py:3: in &lt;module&gt;
    from lxml.html.clean import Cleaner
../../../../Library/Caches/pypoetry/virtualenvs/navigator-document-parser-vnqSsywO-py3.12/lib/python3.12/site-packages/lxml/html/clean.py:18: in &lt;module&gt;
    raise ImportError(
E   ImportError: lxml.html.clean module is now a separate project lxml_html_clean.
E   Install lxml[html_clean] or lxml_html_clean directly.
```

wip
diff --git a/README.md b/README.md
@@ -40,7 +40,7 @@ To run in Docker with the `data/raw` folder as input and `data/processed` as out
 make run_docker
 ```
 
-The CLI operates on an input folder of tasks defined by JSON files in the following format as defined in `cpr_data_access` library dependency. This can be found [here](https://github.com/climatepolicyradar/data-access).
+The CLI operates on an input folder of tasks defined by JSON files in the following format as defined in `cpr_sdk` library dependency. This can be found [here](https://github.com/climatepolicyradar/data-access).
 
 ``` python
 class ParserInput(BaseModel):
diff --git a/cli/parse_htmls.py b/cli/parse_htmls.py
@@ -6,7 +6,7 @@
 
 import cloudpathlib.exceptions
 from cloudpathlib import CloudPath
-from cpr_data_access.parser_models import ParserInput, ParserOutput, HTMLData
+from cpr_sdk.parser_models import ParserInput, ParserOutput, HTMLData
 from tqdm import tqdm
 
 sys.path.append("..")
diff --git a/cli/parse_no_content_type.py b/cli/parse_no_content_type.py
@@ -5,7 +5,7 @@
 
 import cloudpathlib.exceptions
 from cloudpathlib import CloudPath
-from cpr_data_access.parser_models import ParserInput, ParserOutput
+from cpr_sdk.parser_models import ParserInput, ParserOutput
 from tqdm.auto import tqdm
 
 sys.path.append("..")
diff --git a/cli/parse_pdfs.py b/cli/parse_pdfs.py
@@ -17,7 +17,7 @@
 from azure.ai.formrecognizer import AnalyzeResult
 from azure.core.exceptions import ServiceRequestError, HttpResponseError
 from cloudpathlib import CloudPath, S3Path
-from cpr_data_access.parser_models import (
+from cpr_sdk.parser_models import (
     ParserInput,
     ParserOutput,
     PDFData,
diff --git a/cli/run_parser.py b/cli/run_parser.py
@@ -9,7 +9,7 @@
 import click
 import pydantic
 from cloudpathlib import S3Path, CloudPath
-from cpr_data_access.parser_models import (
+from cpr_sdk.parser_models import (
     ParserInput,
     CONTENT_TYPE_HTML,
     CONTENT_TYPE_PDF,
diff --git a/cli/test/test_run_parser.py b/cli/test/test_run_parser.py
@@ -9,13 +9,13 @@
 from azure.core.exceptions import HttpResponseError, ServiceRequestError
 from click.testing import CliRunner
 from cloudpathlib.local import LocalS3Path
-from cpr_data_access.parser_models import (
+from cpr_sdk.parser_models import (
     ParserOutput,
     HTMLData,
     CONTENT_TYPE_HTML,
     CONTENT_TYPE_PDF,
 )
-from cpr_data_access.pipeline_general_models import BackendDocument
+from cpr_sdk.pipeline_general_models import BackendDocument
 from azure_pdf_parser.base import PDFPagesBatchExtracted
 from azure.ai.formrecognizer import AnalyzeResult
 from mock import patch
diff --git a/cli/translate_outputs.py b/cli/translate_outputs.py
@@ -5,7 +5,7 @@
 
 import cloudpathlib.exceptions
 from cloudpathlib import CloudPath
-from cpr_data_access.parser_models import ParserOutput
+from cpr_sdk.parser_models import ParserOutput
 from tqdm.auto import tqdm
 
 from src.config import TARGET_LANGUAGES
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -9,37 +9,38 @@ packages = [
 
 
 [tool.poetry.dependencies]
-python = "^3.10"
+python = ">=3.10,<4.0"
 awscli = "^1.26.16"
-news-please = "^1.5.22"
-pandas = "^1.4.4"
+news-please = "^1.5.48"
+pandas = "^2.2.2"
+numpy = "^2.0.0"
 tqdm = "^4.64.1"
 pydantic = "^2.4.0"
 click = "^8.1.3"
 langdetect = "^1.0.9"
 playwright = "^1.35.0"
 readability-lxml = "^0.8.1"
-bleach = "^5.0.1"
+bleach = "^6.1.0"
 python-json-logger = "^2.0.4"
-cloudpathlib = { extras = ["s3"], version = "^0.10.0" }
+cloudpathlib = { extras = ["s3"], version = "^0.18.1" }
 PyMuPDF = "^1.20.2"
 google-cloud-translate = "^3.8.2"
-psutil = "^5.9.2"
+psutil = "^6.0.0"
 multiprocessing-logging = "^0.3.3"
 json-logging = "^1.3.0"
 tenacity = "^8.2.1"
-pillow = "<10"                                                                                            # to fix import of Image.LINEAR by dependency
+pillow = "^10.4.0"                                                                                            # to fix import of Image.LINEAR by dependency
 azure-ai-formrecognizer = "^3.2.1"
-pytest = "^7.4.0"
 mock = "^5.1.0"
 pypdf2 = "^3.0.1"
-azure-pdf-parser = { git = "https://github.com/climatepolicyradar/azure-pdf-parser.git", tag = "v0.3.1" }
+azure-pdf-parser = { git = "https://github.com/climatepolicyradar/azure-pdf-parser.git", tag = "v0.4.1" }
+lxml = {extras = ["html-clean"], version = "^5.2.2"}
 
 [tool.poetry.dev-dependencies]
-pre-commit = "^2.20.0"
+pre-commit = "^3.7.1"
 pyright = "^1.1.270"
-pytest = "^7.1.3"
-black = "^22.10.0"
+pytest = "^8.3.1"
+black = "^24.4.2"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
diff --git a/src/base.py b/src/base.py
@@ -1,9 +1,10 @@
 """Base classes for parsing."""
+
 import logging
 import logging.config
 from abc import ABC, abstractmethod
 
-from cpr_data_access.parser_models import HTMLData, ParserOutput, ParserInput
+from cpr_sdk.parser_models import HTMLData, ParserOutput, ParserInput
 
 logger = logging.getLogger(__name__)
 
diff --git a/src/html_parser/combined.py b/src/html_parser/combined.py
@@ -2,7 +2,7 @@
 
 import logging
 import requests
-from cpr_data_access.parser_models import ParserInput, ParserOutput
+from cpr_sdk.parser_models import ParserInput, ParserOutput
 from playwright.sync_api import sync_playwright
 from playwright.sync_api._generated import Playwright
 from src.html_parser.newsplease import NewsPleaseParser
diff --git a/src/html_parser/newsplease.py b/src/html_parser/newsplease.py
@@ -7,7 +7,7 @@
 
 from src.config import HTML_HTTP_REQUEST_TIMEOUT, HTML_MIN_NO_LINES_FOR_VALID_TEXT
 from src.base import HTMLParser
-from cpr_data_access.parser_models import (
+from cpr_sdk.parser_models import (
     ParserInput,
     ParserOutput,
     HTMLData,
diff --git a/src/html_parser/readability.py b/src/html_parser/readability.py
@@ -4,7 +4,7 @@
 from typing import List
 import re
 import requests
-from cpr_data_access.parser_models import (
+from cpr_sdk.parser_models import (
     ParserInput,
     ParserOutput,
     HTMLData,
diff --git a/src/html_parser/test/test_parsers.py b/src/html_parser/test/test_parsers.py
@@ -1,5 +1,5 @@
 import pytest
-from cpr_data_access.parser_models import ParserInput, ParserOutput
+from cpr_sdk.parser_models import ParserInput, ParserOutput
 
 from src.base import HTMLParser
 from src.html_parser.newsplease import NewsPleaseParser
diff --git a/src/translator/test/test_translate.py b/src/translator/test/test_translate.py
@@ -2,7 +2,7 @@
 from pathlib import Path
 from unittest import mock
 
-from cpr_data_access.parser_models import ParserOutput
+from cpr_sdk.parser_models import ParserOutput
 from src.translator.translate import translate_parser_output
 
 
diff --git a/src/translator/translate.py b/src/translator/translate.py
@@ -4,7 +4,7 @@
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 
 
-from cpr_data_access.parser_models import ParserOutput
+from cpr_sdk.parser_models import ParserOutput
 from google.cloud import translate_v2
 
 
