diff --git a/python/arcticdb/storage_fixtures/s3.py b/python/arcticdb/storage_fixtures/s3.py index 1822c52234..b251ecbb7e 100644 --- a/python/arcticdb/storage_fixtures/s3.py +++ b/python/arcticdb/storage_fixtures/s3.py @@ -13,6 +13,7 @@ import re import sys import platform +import pprint from tempfile import mkdtemp from urllib.parse import urlparse import boto3 @@ -731,6 +732,26 @@ def run_gcp_server(port, key_file, cert_file): ) +def get_buckets_check(s3_client): + try: + response = s3_client.list_buckets() + buckets = response.get("Buckets", []) + + if buckets: + logger.warning("Buckets found:") + for bucket in buckets: + logger.warning(f"- {bucket['Name']}") + else: + logger.warning("Client is alive, but no buckets exist.") + except botocore.exceptions.EndpointConnectionError: + logger.warning("Could not connect to Moto S3 server. Is it running?") + raise + except botocore.exceptions.ClientError as e: + logger.warning(f"get_buckets_check - Client error: {e.response['Error']['Message']}") + pprint.pprint(e.response) + raise + + def create_bucket(s3_client, bucket_name, max_retries=15): for i in range(max_retries): try: @@ -741,6 +762,10 @@ def create_bucket(s3_client, bucket_name, max_retries=15): raise logger.warning(f"S3 create bucket failed. Retry {1}/{max_retries}") time.sleep(1) + except Exception as e: + logger.error(f"create_bucket - Error: {e.response['Error']['Message']}") + pprint.pprint(e.response) + get_buckets_check(s3_client) class MotoS3StorageFixtureFactory(BaseS3StorageFixtureFactory): @@ -791,10 +816,10 @@ def bucket_name(self, bucket_type="s3"): # We need the unique_id because we have tests that are creating the factory directly # and not using the fixtures # so this guarantees a unique bucket name - return f"test_{bucket_type}_bucket_{self.unique_id}_{self._bucket_id}" + return f"test-{bucket_type}-bucket-{self.unique_id}-{self._bucket_id}" - def _start_server(self): - port = self.port = get_ephemeral_port(2) + def _start_server(self, seed=2): + port = self.port = get_ephemeral_port(seed) self.endpoint = f"{self.http_protocol}://{self.host}:{port}" self.working_dir = mkdtemp(suffix="MotoS3StorageFixtureFactory") self._iam_endpoint = f"{self.http_protocol}://localhost:{port}" @@ -828,15 +853,20 @@ def _start_server(self): wait_for_server_to_come_up(self.endpoint, "moto", self._p, timeout=240) def _safe_enter(self): - for _ in range(3): # For unknown reason, Moto, when running in pytest-xdist, will randomly fail to start + for i in range(5): # For unknown reason, Moto, when running in pytest-xdist, will randomly fail to start try: - self._start_server() + logger.info(f"Attempt to start server - {i}") + self._start_server(2 + i) + self._s3_admin = self._boto(service="s3", key=self.default_key) + get_buckets_check(self._s3_admin) + logger.info(f"Moto S3 STARTED!!! on port {self.port}") break except AssertionError as e: # Thrown by wait_for_server_to_come_up sys.stderr.write(repr(e)) GracefulProcessUtils.terminate(self._p) + except Exception as e: + logger.error(f"Error during startup of Moto S3. Trying again. Error: {e}") - self._s3_admin = self._boto(service="s3", key=self.default_key) return self def __exit__(self, exc_type, exc_value, traceback): @@ -928,8 +958,8 @@ def create_fixture(self) -> NfsS3Bucket: class MotoGcpS3StorageFixtureFactory(MotoS3StorageFixtureFactory): - def _start_server(self): - port = self.port = get_ephemeral_port(3) + def _start_server(self, seed=9): + port = self.port = get_ephemeral_port(seed) self.endpoint = f"{self.http_protocol}://{self.host}:{port}" self.working_dir = mkdtemp(suffix="MotoGcpS3StorageFixtureFactory") self._iam_endpoint = f"{self.http_protocol}://localhost:{port}" diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 1b0f091298..5124ba92d6 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -256,15 +256,21 @@ def s3_no_ssl_storage_factory() -> Generator[MotoS3StorageFixtureFactory, None, @pytest.fixture(scope="session") -def s3_ssl_disabled_storage_factory() -> Generator[MotoS3StorageFixtureFactory, None, None]: - with MotoS3StorageFixtureFactory(use_ssl=False, ssl_test_support=False, bucket_versioning=False) as f: +def s3_ssl_disabled_storage_factory(test_prefix) -> Generator[MotoS3StorageFixtureFactory, None, None]: + with MotoS3StorageFixtureFactory( + use_ssl=False, ssl_test_support=False, bucket_versioning=False, default_prefix=test_prefix + ) as f: yield f @pytest.fixture(scope="session") -def nfs_backed_s3_storage_factory() -> Generator[MotoNfsBackedS3StorageFixtureFactory, None, None]: +def nfs_backed_s3_storage_factory(test_prefix) -> Generator[MotoNfsBackedS3StorageFixtureFactory, None, None]: with MotoNfsBackedS3StorageFixtureFactory( - use_ssl=False, ssl_test_support=False, bucket_versioning=False, _test_only_is_nfs_layout=True + use_ssl=False, + ssl_test_support=False, + bucket_versioning=False, + _test_only_is_nfs_layout=True, + default_prefix=test_prefix, ) as f: yield f @@ -274,13 +280,11 @@ def test_prefix(): return "test_bucket_prefix" -@pytest.fixture(scope="function", params=[MotoNfsBackedS3StorageFixtureFactory, MotoS3StorageFixtureFactory]) -def s3_and_nfs_storage_bucket(test_prefix, request): - with request.param( - use_ssl=False, ssl_test_support=False, bucket_versioning=False, default_prefix=test_prefix - ) as factory: - with factory.create_fixture() as bucket: - yield bucket +@pytest.fixture(scope="function", params=["nfs_backed_s3_storage_factory", "s3_ssl_disabled_storage_factory"]) +def s3_and_nfs_storage_bucket(request): + factory = request.getfixturevalue(request.param) + with factory.create_fixture() as bucket: + yield bucket @pytest.fixture(scope="session") @@ -1626,7 +1630,6 @@ def apply_hybrid_marks(item, source_values: Iterable[str], rules: dict): :param rules: dict of mark_name -> list[str | regex] """ for mark_name, patterns in rules.items(): - # Deduplication guard if item.get_closest_marker(mark_name): continue diff --git a/python/tests/integration/arcticdb/test_s3.py b/python/tests/integration/arcticdb/test_s3.py index c575f3bd19..a3703f1b11 100644 --- a/python/tests/integration/arcticdb/test_s3.py +++ b/python/tests/integration/arcticdb/test_s3.py @@ -196,7 +196,6 @@ def test_wrapped_s3_storage(lib_name, wrapped_s3_storage_bucket): lib.write("s", data=create_df()) -@SKIP_CONDA_MARK # issue with fixture init will be fixed in https://github.com/man-group/ArcticDB/issues/2640 def test_library_get_key_path(lib_name, s3_and_nfs_storage_bucket, test_prefix): lib = s3_and_nfs_storage_bucket.create_version_store_factory(lib_name)() lib.write("s", data=create_df())