Merge pull request #23 from climatepolicyradar/feature/pods-1449-upda…

…te-the-embeddings-generation-to-use-dependabot-for-sdk

cli/test/conftest.py

@@ -4,10 +4,9 @@
 import boto3
 import botocore.client
 import pytest
+from cpr_sdk.parser_models import BlockType, HTMLTextBlock
 from moto import mock_s3
 
-from cpr_data_access.parser_models import BlockType, HTMLTextBlock
-
 
 class S3Client:
     """Helper class to connect to S3 and perform actions on buckets and documents."""

cli/test/test_text2embeddings.py

@@ -6,9 +6,9 @@
 
 import numpy as np
 from click.testing import CliRunner
+from cpr_sdk.parser_models import ParserOutput
 
 from cli.text2embeddings import run_as_cli
-from cpr_data_access.parser_models import ParserOutput
 
 
 def test_run_encoder_local(

pyproject.toml

@@ -17,8 +17,8 @@ boto3 = "^1.26.153"
 aws-error-utils = "^2.7.0"
 moto = "^4.1.11"
 torch = "2.0.0"
-cpr-data-access = {git = "https://github.com/climatepolicyradar/data-access.git", tag = "0.4.0"}
 python-dotenv = "^1.0.1"
+cpr-sdk = "^1.1.6"
 
 [tool.poetry.dev-dependencies]
 black = "^22.1.0"

src/languages.py

@@ -1,8 +1,9 @@
 import logging
 from typing import List
 
+from cpr_sdk.parser_models import ParserOutput
+
 from src import config
-from cpr_data_access.parser_models import ParserOutput
 
 logger = logging.getLogger(__name__)
 

src/test/conftest.py

@@ -5,12 +5,11 @@
 import boto3
 import botocore.client
 import pytest
+from cpr_sdk.parser_models import HTMLData, ParserOutput
+from cpr_sdk.pipeline_general_models import BackendDocument
 from moto import mock_s3
 from pydantic import AnyHttpUrl
 
-from cpr_data_access.parser_models import ParserOutput, HTMLData
-from cpr_data_access.pipeline_general_models import BackendDocument
-
 from cli.test.conftest import get_html_text_block
 
 

src/test/test_languages.py

@@ -1,6 +1,7 @@
 from typing import List
 
-from cpr_data_access.parser_models import ParserOutput
+from cpr_sdk.parser_models import ParserOutput
+
 from src.languages import get_docs_of_supported_language
 
 # TODO test that the warning is logged if the document language is not supported by

src/test/test_utils.py

@@ -1,17 +1,17 @@
 from typing import Sequence
 
 import numpy as np
-from cpr_data_access.parser_models import BlockType, ParserOutput, PDFTextBlock
+from cpr_sdk.parser_models import BlockType, ParserOutput, PDFTextBlock
 
 from cli.test.conftest import test_pdf_file_json  # noqa: F401
 from src import config
 from src.ml import SBERTEncoder
 from src.utils import (
-    filter_on_block_type,
-    replace_text_blocks,
+    encode_parser_output,
     filter_blocks,
+    filter_on_block_type,
     get_ids_with_suffix,
-    encode_parser_output,
+    replace_text_blocks,
 )
 
 

src/utils.py

@@ -1,13 +1,12 @@
 import logging
 import os
 from pathlib import Path
-from typing import Optional, Tuple, Union, List, Set, Sequence
+from typing import List, Optional, Sequence, Set, Tuple, Union
 
 import numpy as np
+from cpr_sdk.parser_models import BlockType, ParserOutput, TextBlock
 
 from src import config
-
-from cpr_data_access.parser_models import ParserOutput, TextBlock, BlockType
 from src.ml import SentenceEncoder
 from src.s3 import get_s3_keys_with_prefix, s3_object_read_text