From 5218f716539b446bb8991bd0c0f389aef30428d8 Mon Sep 17 00:00:00 2001 From: grusev Date: Mon, 15 Sep 2025 13:43:17 +0300 Subject: [PATCH 01/13] test_library_get_key_path check --- python/tests/integration/arcticdb/test_s3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/tests/integration/arcticdb/test_s3.py b/python/tests/integration/arcticdb/test_s3.py index c575f3bd19..a3703f1b11 100644 --- a/python/tests/integration/arcticdb/test_s3.py +++ b/python/tests/integration/arcticdb/test_s3.py @@ -196,7 +196,6 @@ def test_wrapped_s3_storage(lib_name, wrapped_s3_storage_bucket): lib.write("s", data=create_df()) -@SKIP_CONDA_MARK # issue with fixture init will be fixed in https://github.com/man-group/ArcticDB/issues/2640 def test_library_get_key_path(lib_name, s3_and_nfs_storage_bucket, test_prefix): lib = s3_and_nfs_storage_bucket.create_version_store_factory(lib_name)() lib.write("s", data=create_df()) From a440c39960df5084472e70c7507815e7b110a51c Mon Sep 17 00:00:00 2001 From: grusev Date: Thu, 18 Sep 2025 17:09:22 +0300 Subject: [PATCH 02/13] except (botocore.exceptions.EndpointConnectionError, botocore.exceptions.ClientError) as e: --- python/arcticdb/storage_fixtures/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/arcticdb/storage_fixtures/s3.py b/python/arcticdb/storage_fixtures/s3.py index 1822c52234..fcf1bd1804 100644 --- a/python/arcticdb/storage_fixtures/s3.py +++ b/python/arcticdb/storage_fixtures/s3.py @@ -736,7 +736,7 @@ def create_bucket(s3_client, bucket_name, max_retries=15): try: s3_client.create_bucket(Bucket=bucket_name) return - except botocore.exceptions.EndpointConnectionError as e: + except (botocore.exceptions.EndpointConnectionError, botocore.exceptions.ClientError) as e: if i >= max_retries - 1: raise logger.warning(f"S3 create bucket failed. Retry {1}/{max_retries}") From d13d48c61d048b72071343ba62ac5ac7b9df4e39 Mon Sep 17 00:00:00 2001 From: grusev Date: Tue, 23 Sep 2025 09:27:04 +0300 Subject: [PATCH 03/13] use hyphen not underscore in bucket name --- python/arcticdb/storage_fixtures/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/arcticdb/storage_fixtures/s3.py b/python/arcticdb/storage_fixtures/s3.py index fcf1bd1804..b9fa49fe58 100644 --- a/python/arcticdb/storage_fixtures/s3.py +++ b/python/arcticdb/storage_fixtures/s3.py @@ -791,7 +791,7 @@ def bucket_name(self, bucket_type="s3"): # We need the unique_id because we have tests that are creating the factory directly # and not using the fixtures # so this guarantees a unique bucket name - return f"test_{bucket_type}_bucket_{self.unique_id}_{self._bucket_id}" + return f"test-{bucket_type}-bucket-{self.unique_id}-{self._bucket_id}" def _start_server(self): port = self.port = get_ephemeral_port(2) From 1ce0fcb82443d705b39e8ecf0d947ed3b1cfdf72 Mon Sep 17 00:00:00 2001 From: grusev Date: Tue, 23 Sep 2025 15:30:23 +0300 Subject: [PATCH 04/13] More info --- python/arcticdb/storage_fixtures/s3.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/arcticdb/storage_fixtures/s3.py b/python/arcticdb/storage_fixtures/s3.py index b9fa49fe58..35337cec70 100644 --- a/python/arcticdb/storage_fixtures/s3.py +++ b/python/arcticdb/storage_fixtures/s3.py @@ -731,6 +731,23 @@ def run_gcp_server(port, key_file, cert_file): ) +def get_buckets_check(s3_client): + try: + response = s3_client.list_buckets() + buckets = response.get("Buckets", []) + + if buckets: + print("Buckets found:") + for bucket in buckets: + print(f"- {bucket['Name']}") + else: + print("Client is alive, but no buckets exist.") + except botocore.exceptions.EndpointConnectionError: + print("Could not connect to Moto S3 server. Is it running?") + except botocore.exceptions.ClientError as e: + print(f"Client error: {e.response['Error']['Message']}") + + def create_bucket(s3_client, bucket_name, max_retries=15): for i in range(max_retries): try: @@ -740,6 +757,10 @@ def create_bucket(s3_client, bucket_name, max_retries=15): if i >= max_retries - 1: raise logger.warning(f"S3 create bucket failed. Retry {1}/{max_retries}") + logger.warning(f"Error: {e.response['Error']['Message']}") + get_buckets_check() + import pprint + pprint.pprint(e.response) time.sleep(1) From 8237632e63ed3f291afb42f1c2e0766d99a3569a Mon Sep 17 00:00:00 2001 From: grusev Date: Wed, 24 Sep 2025 08:26:53 +0300 Subject: [PATCH 05/13] Added param --- python/arcticdb/storage_fixtures/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/arcticdb/storage_fixtures/s3.py b/python/arcticdb/storage_fixtures/s3.py index 35337cec70..33750e1538 100644 --- a/python/arcticdb/storage_fixtures/s3.py +++ b/python/arcticdb/storage_fixtures/s3.py @@ -758,7 +758,7 @@ def create_bucket(s3_client, bucket_name, max_retries=15): raise logger.warning(f"S3 create bucket failed. Retry {1}/{max_retries}") logger.warning(f"Error: {e.response['Error']['Message']}") - get_buckets_check() + get_buckets_check(s3_client) import pprint pprint.pprint(e.response) time.sleep(1) From d456411c9384c3a72197ac60fdeb5c0c8edbaf06 Mon Sep 17 00:00:00 2001 From: Georgi Rusev Date: Wed, 24 Sep 2025 12:23:48 +0300 Subject: [PATCH 06/13] print to log --- python/arcticdb/storage_fixtures/s3.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/python/arcticdb/storage_fixtures/s3.py b/python/arcticdb/storage_fixtures/s3.py index 33750e1538..2d2f1faf17 100644 --- a/python/arcticdb/storage_fixtures/s3.py +++ b/python/arcticdb/storage_fixtures/s3.py @@ -735,17 +735,17 @@ def get_buckets_check(s3_client): try: response = s3_client.list_buckets() buckets = response.get("Buckets", []) - + if buckets: - print("Buckets found:") + logger.warning("Buckets found:") for bucket in buckets: - print(f"- {bucket['Name']}") + logger.warning(f"- {bucket['Name']}") else: - print("Client is alive, but no buckets exist.") + logger.warning("Client is alive, but no buckets exist.") except botocore.exceptions.EndpointConnectionError: - print("Could not connect to Moto S3 server. Is it running?") + logger.warning("Could not connect to Moto S3 server. Is it running?") except botocore.exceptions.ClientError as e: - print(f"Client error: {e.response['Error']['Message']}") + logger.warning(f"Client error: {e.response['Error']['Message']}") def create_bucket(s3_client, bucket_name, max_retries=15): @@ -760,6 +760,7 @@ def create_bucket(s3_client, bucket_name, max_retries=15): logger.warning(f"Error: {e.response['Error']['Message']}") get_buckets_check(s3_client) import pprint + pprint.pprint(e.response) time.sleep(1) From cadb30cb55c624f53da4b5d56854d4cdb30e7605 Mon Sep 17 00:00:00 2001 From: Georgi Rusev Date: Wed, 24 Sep 2025 13:01:01 +0300 Subject: [PATCH 07/13] try fix --- python/arcticdb/storage_fixtures/s3.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/arcticdb/storage_fixtures/s3.py b/python/arcticdb/storage_fixtures/s3.py index 2d2f1faf17..04db2bddbe 100644 --- a/python/arcticdb/storage_fixtures/s3.py +++ b/python/arcticdb/storage_fixtures/s3.py @@ -850,15 +850,19 @@ def _start_server(self): wait_for_server_to_come_up(self.endpoint, "moto", self._p, timeout=240) def _safe_enter(self): - for _ in range(3): # For unknown reason, Moto, when running in pytest-xdist, will randomly fail to start + for _ in range(5): # For unknown reason, Moto, when running in pytest-xdist, will randomly fail to start try: self._start_server() + self._s3_admin = self._boto(service="s3", key=self.default_key) + get_buckets_check(self._s3_admin) + logger.info("Moto S3 STARTED!!!") break except AssertionError as e: # Thrown by wait_for_server_to_come_up sys.stderr.write(repr(e)) GracefulProcessUtils.terminate(self._p) + except Exception as e: + logger.error(f"Error during startup of Moto S3. Trying again. Error: {e}") - self._s3_admin = self._boto(service="s3", key=self.default_key) return self def __exit__(self, exc_type, exc_value, traceback): From 52c2d02fb6755def38bdad355d634132295170b0 Mon Sep 17 00:00:00 2001 From: Georgi Rusev Date: Wed, 24 Sep 2025 13:16:01 +0300 Subject: [PATCH 08/13] better fix --- python/arcticdb/storage_fixtures/s3.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/arcticdb/storage_fixtures/s3.py b/python/arcticdb/storage_fixtures/s3.py index 04db2bddbe..fe5f2bdd14 100644 --- a/python/arcticdb/storage_fixtures/s3.py +++ b/python/arcticdb/storage_fixtures/s3.py @@ -815,8 +815,8 @@ def bucket_name(self, bucket_type="s3"): # so this guarantees a unique bucket name return f"test-{bucket_type}-bucket-{self.unique_id}-{self._bucket_id}" - def _start_server(self): - port = self.port = get_ephemeral_port(2) + def _start_server(self, seed=2): + port = self.port = get_ephemeral_port(seed) self.endpoint = f"{self.http_protocol}://{self.host}:{port}" self.working_dir = mkdtemp(suffix="MotoS3StorageFixtureFactory") self._iam_endpoint = f"{self.http_protocol}://localhost:{port}" @@ -850,9 +850,9 @@ def _start_server(self): wait_for_server_to_come_up(self.endpoint, "moto", self._p, timeout=240) def _safe_enter(self): - for _ in range(5): # For unknown reason, Moto, when running in pytest-xdist, will randomly fail to start + for i in range(5): # For unknown reason, Moto, when running in pytest-xdist, will randomly fail to start try: - self._start_server() + self._start_server(2 + i) self._s3_admin = self._boto(service="s3", key=self.default_key) get_buckets_check(self._s3_admin) logger.info("Moto S3 STARTED!!!") From 854136a9835099eaf413bb0e78dec6527f6ea982 Mon Sep 17 00:00:00 2001 From: Georgi Rusev Date: Wed, 24 Sep 2025 16:39:47 +0300 Subject: [PATCH 09/13] fix regression for gcp --- python/arcticdb/storage_fixtures/s3.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/arcticdb/storage_fixtures/s3.py b/python/arcticdb/storage_fixtures/s3.py index fe5f2bdd14..a7c8d5fb78 100644 --- a/python/arcticdb/storage_fixtures/s3.py +++ b/python/arcticdb/storage_fixtures/s3.py @@ -13,6 +13,7 @@ import re import sys import platform +import pprint from tempfile import mkdtemp from urllib.parse import urlparse import boto3 @@ -753,16 +754,15 @@ def create_bucket(s3_client, bucket_name, max_retries=15): try: s3_client.create_bucket(Bucket=bucket_name) return - except (botocore.exceptions.EndpointConnectionError, botocore.exceptions.ClientError) as e: + except botocore.exceptions.EndpointConnectionError as e: if i >= max_retries - 1: raise logger.warning(f"S3 create bucket failed. Retry {1}/{max_retries}") - logger.warning(f"Error: {e.response['Error']['Message']}") + time.sleep(1) + except Exception as e: + logger.error(f"Error: {e.response['Error']['Message']}") get_buckets_check(s3_client) - import pprint - pprint.pprint(e.response) - time.sleep(1) class MotoS3StorageFixtureFactory(BaseS3StorageFixtureFactory): @@ -954,8 +954,8 @@ def create_fixture(self) -> NfsS3Bucket: class MotoGcpS3StorageFixtureFactory(MotoS3StorageFixtureFactory): - def _start_server(self): - port = self.port = get_ephemeral_port(3) + def _start_server(self, seed=9): + port = self.port = get_ephemeral_port(seed) self.endpoint = f"{self.http_protocol}://{self.host}:{port}" self.working_dir = mkdtemp(suffix="MotoGcpS3StorageFixtureFactory") self._iam_endpoint = f"{self.http_protocol}://localhost:{port}" From 86f8f2abc9fdcdf182f596de2d0bf8c486e5191c Mon Sep 17 00:00:00 2001 From: Georgi Rusev Date: Wed, 24 Sep 2025 20:00:47 +0300 Subject: [PATCH 10/13] fixes --- python/arcticdb/storage_fixtures/s3.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/python/arcticdb/storage_fixtures/s3.py b/python/arcticdb/storage_fixtures/s3.py index a7c8d5fb78..de5ba629db 100644 --- a/python/arcticdb/storage_fixtures/s3.py +++ b/python/arcticdb/storage_fixtures/s3.py @@ -745,8 +745,11 @@ def get_buckets_check(s3_client): logger.warning("Client is alive, but no buckets exist.") except botocore.exceptions.EndpointConnectionError: logger.warning("Could not connect to Moto S3 server. Is it running?") + raise except botocore.exceptions.ClientError as e: - logger.warning(f"Client error: {e.response['Error']['Message']}") + logger.warning(f"get_buckets_check - Client error: {e.response['Error']['Message']}") + pprint.pprint(e.response) + raise def create_bucket(s3_client, bucket_name, max_retries=15): @@ -760,9 +763,9 @@ def create_bucket(s3_client, bucket_name, max_retries=15): logger.warning(f"S3 create bucket failed. Retry {1}/{max_retries}") time.sleep(1) except Exception as e: - logger.error(f"Error: {e.response['Error']['Message']}") - get_buckets_check(s3_client) + logger.error(f"create_bucket - Error: {e.response['Error']['Message']}") pprint.pprint(e.response) + get_buckets_check(s3_client) class MotoS3StorageFixtureFactory(BaseS3StorageFixtureFactory): @@ -852,10 +855,11 @@ def _start_server(self, seed=2): def _safe_enter(self): for i in range(5): # For unknown reason, Moto, when running in pytest-xdist, will randomly fail to start try: + logger.info(f"Attempt to start server - {i}") self._start_server(2 + i) self._s3_admin = self._boto(service="s3", key=self.default_key) get_buckets_check(self._s3_admin) - logger.info("Moto S3 STARTED!!!") + logger.info(f"Moto S3 STARTED!!! on port {self.port}") break except AssertionError as e: # Thrown by wait_for_server_to_come_up sys.stderr.write(repr(e)) From 97f395c19da8b852c9c4aafee43bdf561e69a80e Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Thu, 25 Sep 2025 11:43:58 +0300 Subject: [PATCH 11/13] Try to make the factory part session based --- python/tests/conftest.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 1b0f091298..3e1cd230aa 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -274,13 +274,11 @@ def test_prefix(): return "test_bucket_prefix" -@pytest.fixture(scope="function", params=[MotoNfsBackedS3StorageFixtureFactory, MotoS3StorageFixtureFactory]) +@pytest.fixture(scope="function", params=[nfs_backed_s3_storage_factory, s3_no_ssl_storage_factory]) def s3_and_nfs_storage_bucket(test_prefix, request): - with request.param( - use_ssl=False, ssl_test_support=False, bucket_versioning=False, default_prefix=test_prefix - ) as factory: - with factory.create_fixture() as bucket: - yield bucket + factory = request.getfixturevalue(request.param) + with factory.create_fixture() as bucket: + yield bucket @pytest.fixture(scope="session") @@ -1626,7 +1624,6 @@ def apply_hybrid_marks(item, source_values: Iterable[str], rules: dict): :param rules: dict of mark_name -> list[str | regex] """ for mark_name, patterns in rules.items(): - # Deduplication guard if item.get_closest_marker(mark_name): continue From cb4b945ac687c46c084d1ca71886553a1ef870ee Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Thu, 25 Sep 2025 12:23:33 +0300 Subject: [PATCH 12/13] Fix fixture error --- python/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 3e1cd230aa..2bd800316b 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -274,7 +274,7 @@ def test_prefix(): return "test_bucket_prefix" -@pytest.fixture(scope="function", params=[nfs_backed_s3_storage_factory, s3_no_ssl_storage_factory]) +@pytest.fixture(scope="function", params=["nfs_backed_s3_storage_factory", "s3_no_ssl_storage_factory"]) def s3_and_nfs_storage_bucket(test_prefix, request): factory = request.getfixturevalue(request.param) with factory.create_fixture() as bucket: From aa76bc9ced79f9ab554a3ea06d4eebde6014eebf Mon Sep 17 00:00:00 2001 From: Georgi Petrov Date: Thu, 25 Sep 2025 13:09:01 +0300 Subject: [PATCH 13/13] Reformat --- python/arcticdb/storage_fixtures/s3.py | 2 +- python/tests/conftest.py | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/python/arcticdb/storage_fixtures/s3.py b/python/arcticdb/storage_fixtures/s3.py index de5ba629db..b251ecbb7e 100644 --- a/python/arcticdb/storage_fixtures/s3.py +++ b/python/arcticdb/storage_fixtures/s3.py @@ -749,7 +749,7 @@ def get_buckets_check(s3_client): except botocore.exceptions.ClientError as e: logger.warning(f"get_buckets_check - Client error: {e.response['Error']['Message']}") pprint.pprint(e.response) - raise + raise def create_bucket(s3_client, bucket_name, max_retries=15): diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 2bd800316b..5124ba92d6 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -256,15 +256,21 @@ def s3_no_ssl_storage_factory() -> Generator[MotoS3StorageFixtureFactory, None, @pytest.fixture(scope="session") -def s3_ssl_disabled_storage_factory() -> Generator[MotoS3StorageFixtureFactory, None, None]: - with MotoS3StorageFixtureFactory(use_ssl=False, ssl_test_support=False, bucket_versioning=False) as f: +def s3_ssl_disabled_storage_factory(test_prefix) -> Generator[MotoS3StorageFixtureFactory, None, None]: + with MotoS3StorageFixtureFactory( + use_ssl=False, ssl_test_support=False, bucket_versioning=False, default_prefix=test_prefix + ) as f: yield f @pytest.fixture(scope="session") -def nfs_backed_s3_storage_factory() -> Generator[MotoNfsBackedS3StorageFixtureFactory, None, None]: +def nfs_backed_s3_storage_factory(test_prefix) -> Generator[MotoNfsBackedS3StorageFixtureFactory, None, None]: with MotoNfsBackedS3StorageFixtureFactory( - use_ssl=False, ssl_test_support=False, bucket_versioning=False, _test_only_is_nfs_layout=True + use_ssl=False, + ssl_test_support=False, + bucket_versioning=False, + _test_only_is_nfs_layout=True, + default_prefix=test_prefix, ) as f: yield f @@ -274,8 +280,8 @@ def test_prefix(): return "test_bucket_prefix" -@pytest.fixture(scope="function", params=["nfs_backed_s3_storage_factory", "s3_no_ssl_storage_factory"]) -def s3_and_nfs_storage_bucket(test_prefix, request): +@pytest.fixture(scope="function", params=["nfs_backed_s3_storage_factory", "s3_ssl_disabled_storage_factory"]) +def s3_and_nfs_storage_bucket(request): factory = request.getfixturevalue(request.param) with factory.create_fixture() as bucket: yield bucket