Feature/pdct 1570 add aws s3 docker image for integration tests (#254)

* Add localstack

* Create S3 bucket in localstack and use in integration tests

* Update .env.example

* Remove unnecessary stuff

* Fix unit tests

* Update docker-compose.yml

* WIP

* WIP

* Remove unnecessary logic

* Update patch version

* Fix unit tests

* Add cleanup fixture that deletes files saved to s3 during integration tests

* Update env example

* Move init-s3 to the tests directory

* Replace hard coded values with env variables and remove unnecessary code from s3 mock

* Rename

* Swallow exceptions if s3 cleanup has error

* Add error handling

* Point localstack to env file

.env.example

@@ -1,19 +1,20 @@
 # DB
 POSTGRES_USER=navigator
 POSTGRES_PASSWORD=password
-ADMIN_POSTGRES_HOST=admin_backend_db
+ADMIN_POSTGRES_HOST=admin_backend_db # for running tests outside docker this needs to be localhost
 PGUSER=navigator
 PGPASSWORD=password
-PGHOST=admin_backend_db
+PGHOST=admin_backend_db # for running tests outside docker this needs to be localhost
 
 # API
 SECRET_KEY=secret_test_key
 API_HOST=http://navigator-admin-backend:8888
 
 # AWS
-BULK_IMPORT_BUCKET=skip
+BULK_IMPORT_BUCKET=bulk-import-bucket
 AWS_ACCESS_KEY_ID=test
-_AWS_SECRET_ACCESS_KEY=test
+AWS_SECRET_ACCESS_KEY=test
+AWS_ENDPOINT_URL=http://localstack:4566 # for running tests outside docker this needs to be http://localhost:4566
 
 # Slack Notifications
 SLACK_WEBHOOK_URL=skip

.gitignore

@@ -108,6 +108,7 @@ celerybeat.pid
 
 # Environments
 .env
+.env.local
 .venv
 env/
 venv/

GETTING_STARTED.md

@@ -19,6 +19,11 @@ Create a `.env` file, you can use the example one running:
 cp .env.example .env
 ```
 
+**NOTE**: When running the app in docker, different values are required for some
+environment variables (as per comments in .env.example). So, if running the app
+or tests without docker, create a separate .env file (e.g. .env.local) with the
+correct values for local development.
+
 This can then be activated in any shell with `pyenv activate admin-backend`.
 
 Also ensure that you have the git commit hooks installed to maintain code
@@ -72,14 +77,15 @@ how the tests run in the Github Actions CI pipeline.
 
 ### Integration Tests
 
-These tests are designed to require a database and therefore will pull and run a
-Postgres container. These can be run locally with
-`pytest -vvv integration_tests` - however this will require that you have spun
-up a local postgres instance (which can be done using `make setup_test_db`).
+These tests are designed to require a database as well as an AWS S3 mock service
+provided via localstack and therefore will pull and run Postgres and localstack
+containers.
+These can be run locally with `pytest -vvv integration_tests` - however this will
+require that you have spun up local postgres and localstack instances.
 
-The preferred way it to use `make setup_test_db integration_tests` as this is
-how the tests run in the Github Actions CI pipeline. These commands were split
-so that the output of the integration tests is easier to read.
+The preferred way it to use `make integration_tests` as this is will build and
+start up all the required services and is how the tests run in the Github Actions
+CI pipeline.
 
 ## Deploying
 

README.md

@@ -21,9 +21,3 @@ See [the linear project](https://linear.app/climate-policy-radar/project/admin-i
 
 If you are new to the repository, please ensure you read the [getting starting guide](GETTING_STARTED.md)
 and the [design guide](DESIGN.md).
-
-## Testing
-
-Currently the integration tests around the ingest endpoint use the AWS S3 mock
-as we do not have our own implementation.
-Ticket to fix that: [PDCT-1570](https://linear.app/climate-policy-radar/issue/PDCT-1570/add-aws-s3-docker-image-for-integration-tests)

app/clients/aws/s3bucket.py

@@ -121,12 +121,6 @@ def upload_ingest_json_to_s3(
 
     filename = f"{ingest_id}-{corpus_import_id}-{current_timestamp}.json"
 
-    if ingest_upload_bucket == "skip":
-        os.makedirs("bulk_import_results", exist_ok=True)
-        with open(f"bulk_import_results/{filename}", "w") as file:
-            json.dump(data, file, indent=4)
-        return
-
     s3_client = boto3.client("s3")
 
     context = S3UploadContext(

docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3.7"
 services:
   admin_backend_db:
     image: postgres:14
@@ -10,11 +9,23 @@ services:
     volumes:
       - admin-data:/var/lib/postgresql/data:cached
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER}"]
+      test: [CMD-SHELL, "pg_isready -U ${POSTGRES_USER}"]
       interval: 5s
       timeout: 3s
       retries: 30
 
+  localstack:
+    container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}"
+    image: localstack/localstack:s3-latest
+    env_file:
+      - .env
+    ports:
+      - 127.0.0.1:4566:4566 # LocalStack Gateway
+    environment:
+      - DEBUG=${DEBUG:-0}
+    volumes:
+      - ./tests/integration_tests/init-s3.py:/etc/localstack/init/ready.d/init-s3.py
+
   navigator-admin-backend:
     build:
       context: ./
@@ -32,6 +43,8 @@ services:
     depends_on:
       admin_backend_db:
         condition: service_healthy
+      localstack:
+        condition: service_started
 
 volumes:
   admin-data:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "admin_backend"
-version = "2.17.16"
+version = "2.17.17"
 description = ""
 authors = ["CPR-dev-team <[email protected]>"]
 packages = [{ include = "app" }, { include = "tests" }]
@@ -54,7 +54,7 @@ env_files = """
     .env.test
     .env
 """
-markers = ["unit"]
+markers = ["unit", "s3"]
 asyncio_mode = "strict"
 
 [tool.pydocstyle]

tests/integration_tests/conftest.py

@@ -1,13 +1,13 @@
+import logging
+import os
 import subprocess
 import tempfile
 from typing import Dict
 
 import boto3
 import pytest
-from botocore.exceptions import ClientError
 from db_client import run_migrations
 from fastapi.testclient import TestClient
-from moto import mock_s3
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy_utils import create_database, database_exists, drop_database
@@ -43,6 +43,9 @@
 from tests.mocks.repos.rollback_event_repo import mock_rollback_event_repo
 from tests.mocks.repos.rollback_family_repo import mock_rollback_family_repo
 
+_LOGGER = logging.getLogger(__name__)
+_LOGGER.setLevel(logging.DEBUG)
+
 CCLW_ORG_ID = 1
 UNFCCC_ORG_ID = 2
 SUPER_ORG_ID = 50
@@ -271,15 +274,22 @@ def invalid_user_header_token() -> Dict[str, str]:
 
 
 @pytest.fixture
-def basic_s3_client():
-    bucket_name = "test_bucket"
-    with mock_s3():
-        conn = boto3.client("s3", region_name="eu-west-2")
-        try:
-            conn.head_bucket(Bucket=bucket_name)
-        except ClientError:
-            conn.create_bucket(
-                Bucket=bucket_name,
-                CreateBucketConfiguration={"LocationConstraint": "eu-west-2"},
-            )
-        yield conn
+def aws_s3_cleanup():
+    # No setup code needed
+    yield
+
+    # Teardown code
+    s3 = boto3.client("s3")
+    try:
+        bucket = os.environ["BULK_IMPORT_BUCKET"]
+        saved_objects = s3.list_objects_v2(Bucket=bucket).get("Contents", None)
+        if saved_objects:
+            for object in saved_objects:
+                s3.delete_object(Bucket=bucket, Key=object["Key"])
+    except Exception as e:
+        _LOGGER.debug(e)
+
+
+def pytest_runtest_setup(item):
+    if "s3" in item.keywords:
+        item.fixturenames.append("aws_s3_cleanup")

tests/integration_tests/ingest/test_ingest.py

@@ -1,7 +1,6 @@
 import logging
-import os
-from unittest.mock import patch
 
+import pytest
 from db_client.models.dfce import FamilyEvent
 from db_client.models.dfce.collection import Collection
 from db_client.models.dfce.family import Family, FamilyDocument
@@ -17,7 +16,6 @@
     default_event,
     default_family,
 )
-from tests.integration_tests.setup_db import setup_db
 
 
 def create_input_json_with_two_of_each_entity():
@@ -55,10 +53,8 @@ def create_input_json_with_two_of_each_entity():
     )
 
 
-@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"})
-def test_ingest_when_ok(
-    data_db: Session, client: TestClient, superuser_header_token, basic_s3_client
-):
+@pytest.mark.s3
+def test_ingest_when_ok(data_db: Session, client: TestClient, superuser_header_token):
     input_json = create_input_json_with_two_of_each_entity()
 
     response = client.post(
@@ -120,16 +116,14 @@ def test_ingest_when_ok(
         assert ev.family_import_id in expected_family_import_ids
 
 
-@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"})
+@pytest.mark.s3
 def test_import_data_rollback(
     caplog,
     data_db: Session,
     client: TestClient,
     superuser_header_token,
     rollback_collection_repo,
-    basic_s3_client,
 ):
-    setup_db(data_db)
     input_json = create_input_json_with_two_of_each_entity()
 
     with caplog.at_level(logging.ERROR):
@@ -153,13 +147,12 @@ def test_import_data_rollback(
     assert actual_collection is None
 
 
-@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"})
+@pytest.mark.s3
 def test_ingest_idempotency(
     caplog,
     data_db: Session,
     client: TestClient,
     superuser_header_token,
-    basic_s3_client,
 ):
     input_json = build_json_file(
         {
@@ -236,13 +229,12 @@ def test_ingest_idempotency(
     )
 
 
-@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"})
+@pytest.mark.s3
 def test_generates_unique_slugs_for_documents_with_identical_titles(
     caplog,
     data_db: Session,
     client: TestClient,
     superuser_header_token,
-    basic_s3_client,
 ):
     """
     This test ensures that given multiple documents with the same title a unique slug
@@ -282,13 +274,12 @@ def test_generates_unique_slugs_for_documents_with_identical_titles(
     )
 
 
-@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"})
+@pytest.mark.s3
 def test_ingest_when_corpus_import_id_invalid(
     caplog,
     data_db: Session,
     client: TestClient,
     superuser_header_token,
-    basic_s3_client,
 ):
     invalid_corpus = "test"
     input_json = create_input_json_with_two_of_each_entity()
@@ -308,13 +299,12 @@ def test_ingest_when_corpus_import_id_invalid(
     assert f"No organisation associated with corpus {invalid_corpus}" in caplog.text
 
 
-@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "test_bucket"})
+@pytest.mark.s3
 def test_ingest_events_when_event_type_invalid(
     caplog,
     data_db: Session,
     client: TestClient,
     superuser_header_token,
-    basic_s3_client,
 ):
 
     input_json = build_json_file(

tests/integration_tests/init-s3.py

@@ -0,0 +1,46 @@
+import logging
+import os
+
+import boto3
+from botocore.exceptions import BotoCoreError, ClientError
+
+_LOGGER = logging.getLogger(__name__)
+_LOGGER.setLevel(logging.DEBUG)
+
+
+def create_s3_bucket() -> None:
+    """
+    Create an S3 bucket using environment variables for configuration.
+
+    Raises:
+        AssertionError: If a required environment variable is missing.
+        BotoCoreError, ClientError: If there's an error with boto3 operation.
+    """
+    required_vars = [
+        "AWS_ENDPOINT_URL",
+        "AWS_ACCESS_KEY_ID",
+        "AWS_SECRET_ACCESS_KEY",
+        "BULK_IMPORT_BUCKET",
+    ]
+    missing_vars = [var for var in required_vars if var not in os.environ]
+    assert (
+        not missing_vars
+    ), f"🔥 Required environment variable(s) missing: {', '.join(missing_vars)}"
+
+    try:
+        s3_client = boto3.client(
+            "s3",
+            endpoint_url=os.environ["AWS_ENDPOINT_URL"],
+            aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+            aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+        )
+
+        s3_client.create_bucket(Bucket=os.environ["BULK_IMPORT_BUCKET"])
+
+        _LOGGER.info("🎉 Bucket created successfully")
+
+    except (BotoCoreError, ClientError) as e:
+        _LOGGER.error(f"🔥 Error: {e}")
+
+
+create_s3_bucket()

tests/unit_tests/clients/aws/test_client.py

@@ -10,7 +10,6 @@
     upload_ingest_json_to_s3,
     upload_json_to_s3,
 )
-from tests.helpers.utils import cleanup_local_files
 
 
 def test_upload_json_to_s3_when_ok(basic_s3_client):
@@ -56,17 +55,3 @@ def test_upload_ingest_json_to_s3_success(basic_s3_client):
     body = get_response["Body"].read().decode("utf-8")
 
     assert json.loads(body) == json_data
-
-
-@patch.dict(os.environ, {"BULK_IMPORT_BUCKET": "skip"})
-def test_do_not_save_ingest_json_to_s3_when_in_local_development(basic_s3_client):
-    json_data = {"test": "test"}
-
-    upload_ingest_json_to_s3("1111-1111", "test_corpus_id", json_data)
-
-    find_response = basic_s3_client.list_objects_v2(
-        Bucket="test_bucket", Prefix="1111-1111-test_corpus_id"
-    )
-
-    assert "Contents" not in find_response
-    cleanup_local_files("bulk_import_results/1111-1111-test_corpus_id*")