Try to make the factory part session based

python/arcticdb/storage_fixtures/s3.py

@@ -13,6 +13,7 @@
 import re
 import sys
 import platform
+import pprint
 from tempfile import mkdtemp
 from urllib.parse import urlparse
 import boto3
@@ -731,6 +732,26 @@ def run_gcp_server(port, key_file, cert_file):
     )
 
 
+def get_buckets_check(s3_client):
+    try:
+        response = s3_client.list_buckets()
+        buckets = response.get("Buckets", [])
+
+        if buckets:
+            logger.warning("Buckets found:")
+            for bucket in buckets:
+                logger.warning(f"- {bucket['Name']}")
+        else:
+            logger.warning("Client is alive, but no buckets exist.")
+    except botocore.exceptions.EndpointConnectionError:
+        logger.warning("Could not connect to Moto S3 server. Is it running?")
+        raise
+    except botocore.exceptions.ClientError as e:
+        logger.warning(f"get_buckets_check - Client error: {e.response['Error']['Message']}")
+        pprint.pprint(e.response)
+        raise
+
+
 def create_bucket(s3_client, bucket_name, max_retries=15):
     for i in range(max_retries):
         try:
@@ -741,6 +762,10 @@ def create_bucket(s3_client, bucket_name, max_retries=15):
                 raise
             logger.warning(f"S3 create bucket failed. Retry {1}/{max_retries}")
             time.sleep(1)
+        except Exception as e:
+            logger.error(f"create_bucket - Error: {e.response['Error']['Message']}")
+            pprint.pprint(e.response)
+            get_buckets_check(s3_client)
 
 
 class MotoS3StorageFixtureFactory(BaseS3StorageFixtureFactory):
@@ -791,10 +816,10 @@ def bucket_name(self, bucket_type="s3"):
         # We need the unique_id because we have tests that are creating the factory directly
         # and not using the fixtures
         # so this guarantees a unique bucket name
-        return f"test_{bucket_type}_bucket_{self.unique_id}_{self._bucket_id}"
+        return f"test-{bucket_type}-bucket-{self.unique_id}-{self._bucket_id}"
 
-    def _start_server(self):
-        port = self.port = get_ephemeral_port(2)
+    def _start_server(self, seed=2):
+        port = self.port = get_ephemeral_port(seed)
         self.endpoint = f"{self.http_protocol}://{self.host}:{port}"
         self.working_dir = mkdtemp(suffix="MotoS3StorageFixtureFactory")
         self._iam_endpoint = f"{self.http_protocol}://localhost:{port}"
@@ -828,15 +853,20 @@ def _start_server(self):
         wait_for_server_to_come_up(self.endpoint, "moto", self._p, timeout=240)
 
     def _safe_enter(self):
-        for _ in range(3):  # For unknown reason, Moto, when running in pytest-xdist, will randomly fail to start
+        for i in range(5):  # For unknown reason, Moto, when running in pytest-xdist, will randomly fail to start
             try:
-                self._start_server()
+                logger.info(f"Attempt to start server - {i}")
+                self._start_server(2 + i)
+                self._s3_admin = self._boto(service="s3", key=self.default_key)
+                get_buckets_check(self._s3_admin)
+                logger.info(f"Moto S3 STARTED!!! on port {self.port}")
                 break
             except AssertionError as e:  # Thrown by wait_for_server_to_come_up
                 sys.stderr.write(repr(e))
                 GracefulProcessUtils.terminate(self._p)
+            except Exception as e:
+                logger.error(f"Error during startup of Moto S3. Trying again. Error: {e}")
 
-        self._s3_admin = self._boto(service="s3", key=self.default_key)
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
@@ -928,8 +958,8 @@ def create_fixture(self) -> NfsS3Bucket:
 
 
 class MotoGcpS3StorageFixtureFactory(MotoS3StorageFixtureFactory):
-    def _start_server(self):
-        port = self.port = get_ephemeral_port(3)
+    def _start_server(self, seed=9):
+        port = self.port = get_ephemeral_port(seed)
         self.endpoint = f"{self.http_protocol}://{self.host}:{port}"
         self.working_dir = mkdtemp(suffix="MotoGcpS3StorageFixtureFactory")
         self._iam_endpoint = f"{self.http_protocol}://localhost:{port}"

python/tests/conftest.py

@@ -256,15 +256,21 @@ def s3_no_ssl_storage_factory() -> Generator[MotoS3StorageFixtureFactory, None,
 
 
 @pytest.fixture(scope="session")
-def s3_ssl_disabled_storage_factory() -> Generator[MotoS3StorageFixtureFactory, None, None]:
-    with MotoS3StorageFixtureFactory(use_ssl=False, ssl_test_support=False, bucket_versioning=False) as f:
+def s3_ssl_disabled_storage_factory(test_prefix) -> Generator[MotoS3StorageFixtureFactory, None, None]:
+    with MotoS3StorageFixtureFactory(
+        use_ssl=False, ssl_test_support=False, bucket_versioning=False, default_prefix=test_prefix
+    ) as f:
         yield f
 
 
 @pytest.fixture(scope="session")
-def nfs_backed_s3_storage_factory() -> Generator[MotoNfsBackedS3StorageFixtureFactory, None, None]:
+def nfs_backed_s3_storage_factory(test_prefix) -> Generator[MotoNfsBackedS3StorageFixtureFactory, None, None]:
     with MotoNfsBackedS3StorageFixtureFactory(
-        use_ssl=False, ssl_test_support=False, bucket_versioning=False, _test_only_is_nfs_layout=True
+        use_ssl=False,
+        ssl_test_support=False,
+        bucket_versioning=False,
+        _test_only_is_nfs_layout=True,
+        default_prefix=test_prefix,
     ) as f:
         yield f
 
@@ -274,13 +280,11 @@ def test_prefix():
     return "test_bucket_prefix"
 
 
-@pytest.fixture(scope="function", params=[MotoNfsBackedS3StorageFixtureFactory, MotoS3StorageFixtureFactory])
-def s3_and_nfs_storage_bucket(test_prefix, request):
-    with request.param(
-        use_ssl=False, ssl_test_support=False, bucket_versioning=False, default_prefix=test_prefix
-    ) as factory:
-        with factory.create_fixture() as bucket:
-            yield bucket
+@pytest.fixture(scope="function", params=["nfs_backed_s3_storage_factory", "s3_ssl_disabled_storage_factory"])
+def s3_and_nfs_storage_bucket(request):
+    factory = request.getfixturevalue(request.param)
+    with factory.create_fixture() as bucket:
+        yield bucket
 
 
 @pytest.fixture(scope="session")
@@ -1626,7 +1630,6 @@ def apply_hybrid_marks(item, source_values: Iterable[str], rules: dict):
     :param rules: dict of mark_name -> list[str | regex]
     """
     for mark_name, patterns in rules.items():
-
         # Deduplication guard
         if item.get_closest_marker(mark_name):
             continue

python/tests/integration/arcticdb/test_s3.py

@@ -196,7 +196,6 @@ def test_wrapped_s3_storage(lib_name, wrapped_s3_storage_bucket):
     lib.write("s", data=create_df())
 
 
-@SKIP_CONDA_MARK  # issue with fixture init will be fixed in https://github.com/man-group/ArcticDB/issues/2640
 def test_library_get_key_path(lib_name, s3_and_nfs_storage_bucket, test_prefix):
     lib = s3_and_nfs_storage_bucket.create_version_store_factory(lib_name)()
     lib.write("s", data=create_df())