Merge pull request #138 from climatepolicyradar/feature/pods-1497-bum…

…p-azure-pdf-parser Bump azure-pdf-parser
climatepolicyradar · Jul 23, 2024 · ad45f95 · ad45f95
2 parents 364966b + edb4579
commit ad45f95
Show file tree

Hide file tree

Showing 16 changed files with 1,237 additions and 483 deletions.
diff --git a/README.md b/README.md
@@ -40,7 +40,7 @@ To run in Docker with the `data/raw` folder as input and `data/processed` as out
 make run_docker
 ```
 
-The CLI operates on an input folder of tasks defined by JSON files in the following format as defined in `cpr_data_access` library dependency. This can be found [here](https://github.com/climatepolicyradar/data-access).
+The CLI operates on an input folder of tasks defined by JSON files in the following format as defined in `cpr_sdk` library dependency. This can be found [here](https://github.com/climatepolicyradar/data-access).
 
 ``` python
 class ParserInput(BaseModel):

diff --git a/cli/parse_htmls.py b/cli/parse_htmls.py
@@ -6,7 +6,7 @@
 
 import cloudpathlib.exceptions
 from cloudpathlib import CloudPath
-from cpr_data_access.parser_models import ParserInput, ParserOutput, HTMLData
+from cpr_sdk.parser_models import ParserInput, ParserOutput, HTMLData
 from tqdm import tqdm
 
 sys.path.append("..")

diff --git a/cli/parse_no_content_type.py b/cli/parse_no_content_type.py
@@ -5,7 +5,7 @@
 
 import cloudpathlib.exceptions
 from cloudpathlib import CloudPath
-from cpr_data_access.parser_models import ParserInput, ParserOutput
+from cpr_sdk.parser_models import ParserInput, ParserOutput
 from tqdm.auto import tqdm
 
 sys.path.append("..")

diff --git a/cli/parse_pdfs.py b/cli/parse_pdfs.py
@@ -17,7 +17,7 @@
 from azure.ai.formrecognizer import AnalyzeResult
 from azure.core.exceptions import ServiceRequestError, HttpResponseError
 from cloudpathlib import CloudPath, S3Path
-from cpr_data_access.parser_models import (
+from cpr_sdk.parser_models import (
     ParserInput,
     ParserOutput,
     PDFData,

diff --git a/cli/run_parser.py b/cli/run_parser.py
@@ -9,7 +9,7 @@
 import click
 import pydantic
 from cloudpathlib import S3Path, CloudPath
-from cpr_data_access.parser_models import (
+from cpr_sdk.parser_models import (
     ParserInput,
     CONTENT_TYPE_HTML,
     CONTENT_TYPE_PDF,

diff --git a/cli/test/test_run_parser.py b/cli/test/test_run_parser.py
@@ -9,13 +9,13 @@
 from azure.core.exceptions import HttpResponseError, ServiceRequestError
 from click.testing import CliRunner
 from cloudpathlib.local import LocalS3Path
-from cpr_data_access.parser_models import (
+from cpr_sdk.parser_models import (
     ParserOutput,
     HTMLData,
     CONTENT_TYPE_HTML,
     CONTENT_TYPE_PDF,
 )
-from cpr_data_access.pipeline_general_models import BackendDocument
+from cpr_sdk.pipeline_general_models import BackendDocument
 from azure_pdf_parser.base import PDFPagesBatchExtracted
 from azure.ai.formrecognizer import AnalyzeResult
 from mock import patch

diff --git a/cli/translate_outputs.py b/cli/translate_outputs.py
@@ -5,7 +5,7 @@
 
 import cloudpathlib.exceptions
 from cloudpathlib import CloudPath
-from cpr_data_access.parser_models import ParserOutput
+from cpr_sdk.parser_models import ParserOutput
 from tqdm.auto import tqdm
 
 from src.config import TARGET_LANGUAGES

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -9,37 +9,38 @@ packages = [
 
 
 [tool.poetry.dependencies]
-python = "^3.10"
+python = ">=3.10,<4.0"
 awscli = "^1.26.16"
-news-please = "^1.5.22"
-pandas = "^1.4.4"
+news-please = "^1.5.48"
+pandas = "^2.2.2"
+numpy = "^2.0.0"
 tqdm = "^4.64.1"
 pydantic = "^2.4.0"
 click = "^8.1.3"
 langdetect = "^1.0.9"
 playwright = "^1.35.0"
 readability-lxml = "^0.8.1"
-bleach = "^5.0.1"
+bleach = "^6.1.0"
 python-json-logger = "^2.0.4"
-cloudpathlib = { extras = ["s3"], version = "^0.10.0" }
+cloudpathlib = { extras = ["s3"], version = "^0.18.1" }
 PyMuPDF = "^1.20.2"
 google-cloud-translate = "^3.8.2"
-psutil = "^5.9.2"
+psutil = "^6.0.0"
 multiprocessing-logging = "^0.3.3"
 json-logging = "^1.3.0"
 tenacity = "^8.2.1"
-pillow = "<10"                                                                                            # to fix import of Image.LINEAR by dependency
+pillow = "^10.4.0"                                                                                            # to fix import of Image.LINEAR by dependency
 azure-ai-formrecognizer = "^3.2.1"
-pytest = "^7.4.0"
 mock = "^5.1.0"
 pypdf2 = "^3.0.1"
-azure-pdf-parser = { git = "https://github.com/climatepolicyradar/azure-pdf-parser.git", tag = "v0.3.1" }
+azure-pdf-parser = { git = "https://github.com/climatepolicyradar/azure-pdf-parser.git", tag = "v0.4.1" }
+lxml = {extras = ["html-clean"], version = "^5.2.2"}
 
 [tool.poetry.dev-dependencies]
-pre-commit = "^2.20.0"
+pre-commit = "^3.7.1"
 pyright = "^1.1.270"
-pytest = "^7.1.3"
-black = "^22.10.0"
+pytest = "^8.3.1"
+black = "^24.4.2"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

diff --git a/src/base.py b/src/base.py
@@ -1,9 +1,10 @@
 """Base classes for parsing."""
+
 import logging
 import logging.config
 from abc import ABC, abstractmethod
 
-from cpr_data_access.parser_models import HTMLData, ParserOutput, ParserInput
+from cpr_sdk.parser_models import HTMLData, ParserOutput, ParserInput
 
 logger = logging.getLogger(__name__)
 

diff --git a/src/html_parser/combined.py b/src/html_parser/combined.py
@@ -2,7 +2,7 @@
 
 import logging
 import requests
-from cpr_data_access.parser_models import ParserInput, ParserOutput
+from cpr_sdk.parser_models import ParserInput, ParserOutput
 from playwright.sync_api import sync_playwright
 from playwright.sync_api._generated import Playwright
 from src.html_parser.newsplease import NewsPleaseParser

diff --git a/src/html_parser/newsplease.py b/src/html_parser/newsplease.py
@@ -7,7 +7,7 @@
 
 from src.config import HTML_HTTP_REQUEST_TIMEOUT, HTML_MIN_NO_LINES_FOR_VALID_TEXT
 from src.base import HTMLParser
-from cpr_data_access.parser_models import (
+from cpr_sdk.parser_models import (
     ParserInput,
     ParserOutput,
     HTMLData,

diff --git a/src/html_parser/readability.py b/src/html_parser/readability.py
@@ -4,7 +4,7 @@
 from typing import List
 import re
 import requests
-from cpr_data_access.parser_models import (
+from cpr_sdk.parser_models import (
     ParserInput,
     ParserOutput,
     HTMLData,

diff --git a/src/html_parser/test/test_parsers.py b/src/html_parser/test/test_parsers.py
@@ -1,5 +1,5 @@
 import pytest
-from cpr_data_access.parser_models import ParserInput, ParserOutput
+from cpr_sdk.parser_models import ParserInput, ParserOutput
 
 from src.base import HTMLParser
 from src.html_parser.newsplease import NewsPleaseParser

diff --git a/src/translator/test/test_translate.py b/src/translator/test/test_translate.py
@@ -2,7 +2,7 @@
 from pathlib import Path
 from unittest import mock
 
-from cpr_data_access.parser_models import ParserOutput
+from cpr_sdk.parser_models import ParserOutput
 from src.translator.translate import translate_parser_output
 
 

diff --git a/src/translator/translate.py b/src/translator/translate.py
@@ -4,7 +4,7 @@
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 
 
-from cpr_data_access.parser_models import ParserOutput
+from cpr_sdk.parser_models import ParserOutput
 from google.cloud import translate_v2