From 31df878d537b81fe9d118443631e91d3fe3a98ea Mon Sep 17 00:00:00 2001 From: Carlos Villavicencio <123113322+carlos-villavicencio-adsk@users.noreply.github.com> Date: Mon, 9 Dec 2024 12:52:42 -0500 Subject: [PATCH] SG-36677 Optimize payload by prevent unnecessary data (#360) * Optimize payload by prevent unnecessary data * Packaging for pre-release * Code review improvements * Improve example * Improve example * Process env var * Format documentation code * Swap env var logic * Restrict env var values * Remove lowercase transform * Read env var once * Add unit tests * Packaging for v3.7.0 --- HISTORY.rst | 7 +++++ docs/reference.rst | 30 +++++++++++++++++++++ setup.py | 11 ++------ shotgun_api3/shotgun.py | 29 +++++++++++++++++++-- tests/test_unit.py | 58 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 124 insertions(+), 11 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 92bf444d..ccfc44e3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,13 @@ Flow Production Tracking Python API Changelog Here you can see the full list of changes between each Python API release. +v3.7.0 (2024 Dec 9) +=========================== +- Remove unnecessary data in the payload when combining related queries before sending it to the server. + This would improve overall performance decreasing network latency and server processing. + See documentation for more information. + + v3.6.2 (2024 Aug 13) ==================== - Remove Ticket entity reference and prepare this to run in CI. diff --git a/docs/reference.rst b/docs/reference.rst index da171ea8..6304fc09 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -949,6 +949,36 @@ Stores the number of milliseconds to wait between request retries. By default, In the case that both this environment variable and the config's ``rpc_attempt_interval`` property are set, the value in ``rpc_attempt_interal`` will be used. + +SHOTGUN_API_ENABLE_ENTITY_OPTIMIZATION +======================================= + +.. note:: (v3.7.0) This is an experimental feature. Feel free to disable this feature if you are experiencing any issues. + +When set to ``1``, this environment variable will enable the entity optimization feature. +This feature is disabled by default and is used to reduce the payload size made to the server when retrieving entities +improving overall performance by decreasing network latency and server processing. + +For example, a ``find`` call like this: + +.. code-block:: python + + sg.find('Asset', [['project', 'is', { + 'created_at': datetime.datetime(2015, 12, 16, 11, 2, 10, tzinfo), + 'id': 9999, + 'name': 'Demo: Game', + 'type': 'Project', + # More entity attributes + }]]) + + +Will internally be transformed as if you invoked something like this: + +.. code-block:: python + + sg.find('Asset', [['project', 'is', {'id': 999, 'type': 'Project'}]]) + + ************ Localization ************ diff --git a/setup.py b/setup.py index c46fff40..3305fd5a 100644 --- a/setup.py +++ b/setup.py @@ -18,16 +18,9 @@ f = open('LICENSE') license = f.read().strip() -# For python 2.4 support -script_args = sys.argv[1:] -if (sys.version_info[0] <= 2) or (sys.version_info[0] == 2 and sys.version_info[1] <= 5): - if 'install' in script_args and '--no-compile' not in script_args: - script_args.append('--no-compile') - - setup( name='shotgun_api3', - version='3.6.2', + version='3.7.0', description='Flow Production Tracking Python API', long_description=readme, author='Autodesk', @@ -35,7 +28,7 @@ url='https://github.com/shotgunsoftware/python-api', license=license, packages=find_packages(exclude=('tests',)), - script_args=script_args, + script_args=sys.argv[1:], include_package_data=True, package_data={'': ['cacerts.txt', 'cacert.pem']}, zip_safe=False, diff --git a/shotgun_api3/shotgun.py b/shotgun_api3/shotgun.py index 99b2a595..87f68d3e 100644 --- a/shotgun_api3/shotgun.py +++ b/shotgun_api3/shotgun.py @@ -105,6 +105,8 @@ def _is_mimetypes_broken(): SG_TIMEZONE = SgTimezone() +SHOTGUN_API_ENABLE_ENTITY_OPTIMIZATION = False + NO_SSL_VALIDATION = False """ Turns off hostname matching validation for SSL certificates @@ -116,7 +118,7 @@ def _is_mimetypes_broken(): # ---------------------------------------------------------------------------- # Version -__version__ = "3.6.2" +__version__ = "3.7.0" # ---------------------------------------------------------------------------- # Errors @@ -649,7 +651,11 @@ def __init__(self, if self.config.rpc_attempt_interval < 0: raise ValueError("Value of SHOTGUN_API_RETRY_INTERVAL must be positive, " "got '%s'." % self.config.rpc_attempt_interval) - + + global SHOTGUN_API_ENABLE_ENTITY_OPTIMIZATION + if os.environ.get("SHOTGUN_API_ENABLE_ENTITY_OPTIMIZATION", "0").strip().lower() == "1": + SHOTGUN_API_ENABLE_ENTITY_OPTIMIZATION = True + self._connection = None self.__ca_certs = self._get_certs_file(ca_certs) @@ -4470,6 +4476,25 @@ def _translate_filters_simple(sg_filter): if len(values) == 1 and isinstance(values[0], (list, tuple)): values = values[0] + # Payload optimization: Do not send a full object + # just send the `type` and `id` when combining related queries + global SHOTGUN_API_ENABLE_ENTITY_OPTIMIZATION + if ( + SHOTGUN_API_ENABLE_ENTITY_OPTIMIZATION + and condition["path"] != "id" + and condition["relation"] in ["is", "is_not"] + and isinstance(values[0], dict) + ): + try: + values = [ + { + "type": values[0]["type"], + "id": values[0]["id"], + } + ] + except KeyError: + pass + condition["values"] = values return condition diff --git a/tests/test_unit.py b/tests/test_unit.py index 84bd35b6..096ca932 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -405,6 +405,63 @@ def test_invalid(self): self.assertRaises(api.ShotgunError, api.shotgun._translate_filters, filters, "all") + def test_related_object(self): + filters = [ + [ + "project", + "is", + {"foo": "foo", "bar": "bar", "id": 999, "baz": "baz", "type": "Anything"}, + ], + ] + expected = { + "logical_operator": "and", + "conditions": [ + { + "path": "project", + "relation": "is", + "values": [ + { + "foo": "foo", + "bar": "bar", + "baz": "baz", + "id": 999, + "type": "Anything", + } + ], + } + ], + } + result = api.shotgun._translate_filters(filters, "all") + self.assertEqual(result, expected) + + def test_related_object_entity_optimization(self): + filters = [ + [ + "project", + "is", + {"foo": "foo", "bar": "bar", "id": 999, "baz": "baz", "type": "Anything"}, + ], + ] + expected = { + "logical_operator": "and", + "conditions": [ + { + "path": "project", + "relation": "is", + "values": [ + { + "id": 999, + "type": "Anything", + } + ], + } + ], + } + os.environ["SHOTGUN_API_ENABLE_ENTITY_OPTIMIZATION"] = "1" + api.Shotgun("http://server_path", "script_name", "api_key", connect=False) + result = api.shotgun._translate_filters(filters, "all") + self.assertEqual(result, expected) + class TestCerts(unittest.TestCase): # A dummy bad url provided by Amazon @@ -506,5 +563,6 @@ def _test_mimetypes_import(self, platform, major, minor, patch_number, result, m mock.platform = platform self.assertEqual(_is_mimetypes_broken(), result) + if __name__ == '__main__': unittest.main()