From 5434d08d5cc55dd98ce91bb984f9510cb30c47fc Mon Sep 17 00:00:00 2001 From: Anant Simran Singh Date: Tue, 26 Sep 2023 22:10:42 +0000 Subject: [PATCH 1/5] Add tcp keep alive --- tests/.wickerconfig.test.json | 3 ++- tests/test_s3_storage.py | 2 +- wicker/core/config.py | 2 ++ wicker/core/storage.py | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/.wickerconfig.test.json b/tests/.wickerconfig.test.json index c78185a..939c79e 100644 --- a/tests/.wickerconfig.test.json +++ b/tests/.wickerconfig.test.json @@ -5,7 +5,8 @@ "boto_config": { "max_pool_connections":10, "read_timeout_s": 140, - "connect_timeout_s": 140 + "connect_timeout_s": 140, + "tcp_keepalive": true } }, "dynamodb_config": { diff --git a/tests/test_s3_storage.py b/tests/test_s3_storage.py index 2095e75..9291851 100644 --- a/tests/test_s3_storage.py +++ b/tests/test_s3_storage.py @@ -147,7 +147,7 @@ def test_get_column_concatenated_bytes_files_path(self, mock_get_config: mock.Mo "aws_s3_config": { "s3_datasets_path": "s3://dummy_bucket/wicker/", "region": "us-east-1", - "boto_config": {"max_pool_connections": 10, "read_timeout_s": 140, "connect_timeout_s": 140}, + "boto_config": {"max_pool_connections": 10, "read_timeout_s": 140, "connect_timeout_s": 140, "tcp_keepalive": True}, }, "dynamodb_config": {"table_name": "fake-table-name", "region": "us-west-2"}, "storage_download_config": { diff --git a/wicker/core/config.py b/wicker/core/config.py index 49d4c1d..8344464 100644 --- a/wicker/core/config.py +++ b/wicker/core/config.py @@ -28,6 +28,7 @@ class BotoS3Config: max_pool_connections: int read_timeout_s: int connect_timeout_s: int + tcp_keepalive: bool @classmethod def from_json(cls, data: Dict[str, Any]) -> BotoS3Config: @@ -35,6 +36,7 @@ def from_json(cls, data: Dict[str, Any]) -> BotoS3Config: max_pool_connections=data["max_pool_connections"], read_timeout_s=data["read_timeout_s"], connect_timeout_s=data["connect_timeout_s"], + tcp_keepalive=data["tcp_keepalive"] ) diff --git a/wicker/core/storage.py b/wicker/core/storage.py index 24fb8c5..db86e2b 100644 --- a/wicker/core/storage.py +++ b/wicker/core/storage.py @@ -42,6 +42,7 @@ def __init__(self, session: Optional[boto3.session.Session] = None) -> None: max_pool_connections=boto_config.max_pool_connections, read_timeout=boto_config.read_timeout_s, connect_timeout=boto_config.connect_timeout_s, + tcp_keepalive= boto_config.tcp_keepalive ) self.session = boto3.session.Session() if session is None else session self.client = self.session.client("s3", config=boto_client_config) From 4cbaa8f232f175da1e554285fe70a96745e5cf64 Mon Sep 17 00:00:00 2001 From: Anant Simran Singh Date: Tue, 26 Sep 2023 22:19:37 +0000 Subject: [PATCH 2/5] Add version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 429d94a..b0a1227 100755 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.9 \ No newline at end of file +0.0.10 \ No newline at end of file From 6e54118906f57feb15413839fc339577a38cec34 Mon Sep 17 00:00:00 2001 From: Anant Simran Singh Date: Thu, 28 Sep 2023 02:15:38 +0000 Subject: [PATCH 3/5] disable tcp keep alive by default --- tests/.wickerconfig.test.json | 2 +- tests/test_s3_storage.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/.wickerconfig.test.json b/tests/.wickerconfig.test.json index 939c79e..5c7e6cb 100644 --- a/tests/.wickerconfig.test.json +++ b/tests/.wickerconfig.test.json @@ -6,7 +6,7 @@ "max_pool_connections":10, "read_timeout_s": 140, "connect_timeout_s": 140, - "tcp_keepalive": true + "tcp_keepalive": false } }, "dynamodb_config": { diff --git a/tests/test_s3_storage.py b/tests/test_s3_storage.py index 9291851..5a00797 100644 --- a/tests/test_s3_storage.py +++ b/tests/test_s3_storage.py @@ -147,7 +147,7 @@ def test_get_column_concatenated_bytes_files_path(self, mock_get_config: mock.Mo "aws_s3_config": { "s3_datasets_path": "s3://dummy_bucket/wicker/", "region": "us-east-1", - "boto_config": {"max_pool_connections": 10, "read_timeout_s": 140, "connect_timeout_s": 140, "tcp_keepalive": True}, + "boto_config": {"max_pool_connections": 10, "read_timeout_s": 140, "connect_timeout_s": 140, "tcp_keepalive": False}, }, "dynamodb_config": {"table_name": "fake-table-name", "region": "us-west-2"}, "storage_download_config": { From ee473ccb580bddbfa7430793f5f02882fe1457ee Mon Sep 17 00:00:00 2001 From: Anant Simran Singh Date: Thu, 28 Sep 2023 19:00:39 +0000 Subject: [PATCH 4/5] change default value --- tests/.wickerconfig.test.json | 2 +- tests/test_s3_storage.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/.wickerconfig.test.json b/tests/.wickerconfig.test.json index 5c7e6cb..939c79e 100644 --- a/tests/.wickerconfig.test.json +++ b/tests/.wickerconfig.test.json @@ -6,7 +6,7 @@ "max_pool_connections":10, "read_timeout_s": 140, "connect_timeout_s": 140, - "tcp_keepalive": false + "tcp_keepalive": true } }, "dynamodb_config": { diff --git a/tests/test_s3_storage.py b/tests/test_s3_storage.py index 5a00797..9291851 100644 --- a/tests/test_s3_storage.py +++ b/tests/test_s3_storage.py @@ -147,7 +147,7 @@ def test_get_column_concatenated_bytes_files_path(self, mock_get_config: mock.Mo "aws_s3_config": { "s3_datasets_path": "s3://dummy_bucket/wicker/", "region": "us-east-1", - "boto_config": {"max_pool_connections": 10, "read_timeout_s": 140, "connect_timeout_s": 140, "tcp_keepalive": False}, + "boto_config": {"max_pool_connections": 10, "read_timeout_s": 140, "connect_timeout_s": 140, "tcp_keepalive": True}, }, "dynamodb_config": {"table_name": "fake-table-name", "region": "us-west-2"}, "storage_download_config": { From d0ee18147315ecae6580787cbaffbd7cca190964 Mon Sep 17 00:00:00 2001 From: Anant Simran Singh Date: Thu, 28 Sep 2023 19:00:55 +0000 Subject: [PATCH 5/5] fix lint --- tests/test_s3_storage.py | 7 ++++++- wicker/core/config.py | 2 +- wicker/core/storage.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_s3_storage.py b/tests/test_s3_storage.py index 9291851..18ab046 100644 --- a/tests/test_s3_storage.py +++ b/tests/test_s3_storage.py @@ -147,7 +147,12 @@ def test_get_column_concatenated_bytes_files_path(self, mock_get_config: mock.Mo "aws_s3_config": { "s3_datasets_path": "s3://dummy_bucket/wicker/", "region": "us-east-1", - "boto_config": {"max_pool_connections": 10, "read_timeout_s": 140, "connect_timeout_s": 140, "tcp_keepalive": True}, + "boto_config": { + "max_pool_connections": 10, + "read_timeout_s": 140, + "connect_timeout_s": 140, + "tcp_keepalive": True, + }, }, "dynamodb_config": {"table_name": "fake-table-name", "region": "us-west-2"}, "storage_download_config": { diff --git a/wicker/core/config.py b/wicker/core/config.py index 8344464..b8a9fc7 100644 --- a/wicker/core/config.py +++ b/wicker/core/config.py @@ -36,7 +36,7 @@ def from_json(cls, data: Dict[str, Any]) -> BotoS3Config: max_pool_connections=data["max_pool_connections"], read_timeout_s=data["read_timeout_s"], connect_timeout_s=data["connect_timeout_s"], - tcp_keepalive=data["tcp_keepalive"] + tcp_keepalive=data["tcp_keepalive"], ) diff --git a/wicker/core/storage.py b/wicker/core/storage.py index db86e2b..24e00a8 100644 --- a/wicker/core/storage.py +++ b/wicker/core/storage.py @@ -42,7 +42,7 @@ def __init__(self, session: Optional[boto3.session.Session] = None) -> None: max_pool_connections=boto_config.max_pool_connections, read_timeout=boto_config.read_timeout_s, connect_timeout=boto_config.connect_timeout_s, - tcp_keepalive= boto_config.tcp_keepalive + tcp_keepalive=boto_config.tcp_keepalive, ) self.session = boto3.session.Session() if session is None else session self.client = self.session.client("s3", config=boto_client_config)