[s3] Add client_config configuration setting (#1386)

docs/backends/amazon-S3.rst

@@ -242,12 +242,25 @@ Settings
     The signature versions are not backwards compatible so be careful about url endpoints if making this change
     for legacy projects.
 
+``client_config`` or ``AWS_S3_CLIENT_CONFIG``
+
+  Default: ``None``
+
+  An instance of ``botocore.config.Config`` to do advanced configuration of the client such as
+  ``max_pool_connections``. See all options in the `Botocore docs`_.
+
+  .. note::
+
+    Setting this overrides the settings for ``addressing_style``, ``signature_version`` and
+    ``proxies``. Include them as arguments to your ``botocore.config.Config`` class if you need them.
+
 .. _AWS Signature Version 4: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
 .. _S3 region list: https://docs.aws.amazon.com/general/latest/gr/s3.html#s3_region
 .. _list of canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 .. _Boto3 docs for uploading files: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.put_object
 .. _Boto3 docs for TransferConfig: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.TransferConfig
 .. _ManifestStaticFilesStorage: https://docs.djangoproject.com/en/3.1/ref/contrib/staticfiles/#manifeststaticfilesstorage
+.. _Botocore docs: https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html#botocore.config.Config
 
 .. _cloudfront-signed-url-header:
 

storages/backends/s3.py

@@ -35,7 +35,7 @@
     import boto3.session
     import s3transfer.constants
     from boto3.s3.transfer import TransferConfig
-    from botocore.client import Config
+    from botocore.config import Config
     from botocore.exceptions import ClientError
     from botocore.signers import CloudFrontSigner
 except ImportError as e:
@@ -318,8 +318,17 @@ def __init__(self, **settings):
         self._bucket = None
         self._connections = threading.local()
 
-        if not self.config:
-            self.config = Config(
+        if self.config is not None:
+            warnings.warn(
+                "The 'config' class property is deprecated and will be "
+                "removed in a future version. Use AWS_S3_CLIENT_CONFIG "
+                "to customize any of the botocore.config.Config parameters.",
+                DeprecationWarning,
+            )
+            self.client_config = self.config
+
+        if self.client_config is None:
+            self.client_config = Config(
                 s3={"addressing_style": self.addressing_style},
                 signature_version=self.signature_version,
                 proxies=self.proxies,
@@ -407,6 +416,7 @@ def get_default_settings(self):
             "default_acl": setting("AWS_DEFAULT_ACL", None),
             "use_threads": setting("AWS_S3_USE_THREADS", True),
             "transfer_config": setting("AWS_S3_TRANSFER_CONFIG", None),
+            "client_config": setting("AWS_S3_CLIENT_CONFIG", None),
         }
 
     def __getstate__(self):
@@ -430,7 +440,7 @@ def connection(self):
                 region_name=self.region_name,
                 use_ssl=self.use_ssl,
                 endpoint_url=self.endpoint_url,
-                config=self.config,
+                config=self.client_config,
                 verify=self.verify,
             )
         return self._connections.connection

tests/test_s3.py

@@ -11,6 +11,7 @@
 
 import boto3
 import boto3.s3.transfer
+from botocore.config import Config as ClientConfig
 from botocore.exceptions import ClientError
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
@@ -42,6 +43,18 @@ def test_s3_session(self):
                 _ = storage.connection
                 mock_session.assert_called_once_with(profile_name="test_profile")
 
+    def test_client_config(self):
+        with override_settings(
+            AWS_S3_CLIENT_CONFIG=ClientConfig(max_pool_connections=30)
+        ):
+            storage = s3.S3Storage()
+            with mock.patch("boto3.Session.resource") as mock_resource:
+                _ = storage.connection
+                mock_resource.assert_called_once()
+                self.assertEqual(
+                    30, mock_resource.call_args[1]["config"].max_pool_connections
+                )
+
     def test_pickle_with_bucket(self):
         """
         Test that the storage can be pickled with a bucket attached