SG-36677 Optimize payload by prevent unnecessary data (#360)

* Optimize payload by prevent unnecessary data

* Packaging for pre-release

* Code review improvements

* Improve example

* Improve example

* Process env var

* Format documentation code

* Swap env var logic

* Restrict env var values

* Remove lowercase transform

* Read env var once

* Add unit tests

* Packaging for v3.7.0

Author: carlos-villavicencio-adsk

HISTORY.rst

@@ -4,6 +4,13 @@ Flow Production Tracking Python API Changelog
 
 Here you can see the full list of changes between each Python API release.
 
+v3.7.0 (2024 Dec 9)
+===========================
+- Remove unnecessary data in the payload when combining related queries before sending it to the server.
+  This would improve overall performance decreasing network latency and server processing.
+  See documentation for more information.
+
+
 v3.6.2 (2024 Aug 13)
 ====================
 - Remove Ticket entity reference and prepare this to run in CI.

docs/reference.rst

@@ -949,6 +949,36 @@ Stores the number of milliseconds to wait between request retries.  By default,
 
 In the case that both this environment variable and the config's ``rpc_attempt_interval`` property are set, the value in ``rpc_attempt_interal`` will be used.
 
+
+SHOTGUN_API_ENABLE_ENTITY_OPTIMIZATION
+=======================================
+
+.. note:: (v3.7.0) This is an experimental feature. Feel free to disable this feature if you are experiencing any issues.
+
+When set to ``1``, this environment variable will enable the entity optimization feature.
+This feature is disabled by default and is used to reduce the payload size made to the server when retrieving entities
+improving overall performance by decreasing network latency and server processing.
+
+For example, a ``find`` call like this:
+
+.. code-block:: python
+
+    sg.find('Asset', [['project', 'is', {
+        'created_at': datetime.datetime(2015, 12, 16, 11, 2, 10, tzinfo),
+        'id': 9999,
+        'name': 'Demo: Game',
+        'type': 'Project',
+        # More entity attributes
+    }]])
+
+
+Will internally be transformed as if you invoked something like this:
+
+.. code-block:: python
+
+    sg.find('Asset', [['project', 'is', {'id': 999, 'type': 'Project'}]]) 
+
+
 ************
 Localization
 ************

setup.py

@@ -18,24 +18,17 @@
 f = open('LICENSE')
 license = f.read().strip()
 
-# For python 2.4 support
-script_args = sys.argv[1:]
-if (sys.version_info[0] <= 2) or (sys.version_info[0] == 2 and sys.version_info[1] <= 5):
-    if 'install' in script_args and '--no-compile' not in script_args:
-        script_args.append('--no-compile')
-
-
 setup(
     name='shotgun_api3',
-    version='3.6.2',
+    version='3.7.0',
     description='Flow Production Tracking Python API',
     long_description=readme,
     author='Autodesk',
     author_email='https://www.autodesk.com/support/contact-support',
     url='https://github.com/shotgunsoftware/python-api',
     license=license,
     packages=find_packages(exclude=('tests',)),
-    script_args=script_args,
+    script_args=sys.argv[1:],
     include_package_data=True,
     package_data={'': ['cacerts.txt', 'cacert.pem']},
     zip_safe=False,

shotgun_api3/shotgun.py

@@ -105,6 +105,8 @@ def _is_mimetypes_broken():
 
 SG_TIMEZONE = SgTimezone()
 
+SHOTGUN_API_ENABLE_ENTITY_OPTIMIZATION = False
+
 NO_SSL_VALIDATION = False
 """
 Turns off hostname matching validation for SSL certificates
@@ -116,7 +118,7 @@ def _is_mimetypes_broken():
 
 # ----------------------------------------------------------------------------
 # Version
-__version__ = "3.6.2"
+__version__ = "3.7.0"
 
 # ----------------------------------------------------------------------------
 # Errors
@@ -649,7 +651,11 @@ def __init__(self,
         if self.config.rpc_attempt_interval < 0:
             raise ValueError("Value of SHOTGUN_API_RETRY_INTERVAL must be positive, "
                              "got '%s'." % self.config.rpc_attempt_interval)
-
+        
+        global SHOTGUN_API_ENABLE_ENTITY_OPTIMIZATION
+        if os.environ.get("SHOTGUN_API_ENABLE_ENTITY_OPTIMIZATION", "0").strip().lower() == "1":
+            SHOTGUN_API_ENABLE_ENTITY_OPTIMIZATION = True
+            
         self._connection = None
 
         self.__ca_certs = self._get_certs_file(ca_certs)
@@ -4470,6 +4476,25 @@ def _translate_filters_simple(sg_filter):
     if len(values) == 1 and isinstance(values[0], (list, tuple)):
         values = values[0]
 
+    # Payload optimization: Do not send a full object
+    # just send the `type` and `id` when combining related queries
+    global SHOTGUN_API_ENABLE_ENTITY_OPTIMIZATION
+    if (
+        SHOTGUN_API_ENABLE_ENTITY_OPTIMIZATION
+        and condition["path"] != "id"
+        and condition["relation"] in ["is", "is_not"]
+        and isinstance(values[0], dict)
+    ):
+        try:
+            values = [
+                {
+                    "type": values[0]["type"],
+                    "id": values[0]["id"],
+                }
+            ]
+        except KeyError:
+            pass
+
     condition["values"] = values
 
     return condition

tests/test_unit.py

@@ -405,6 +405,63 @@ def test_invalid(self):
 
         self.assertRaises(api.ShotgunError, api.shotgun._translate_filters, filters, "all")
 
+    def test_related_object(self):
+        filters = [
+            [
+                "project",
+                "is",
+                {"foo": "foo", "bar": "bar", "id": 999, "baz": "baz", "type": "Anything"},
+            ],
+        ]
+        expected = {
+            "logical_operator": "and",
+            "conditions": [
+                {
+                    "path": "project",
+                    "relation": "is",
+                    "values": [
+                        {
+                            "foo": "foo",
+                            "bar": "bar",
+                            "baz": "baz",
+                            "id": 999,
+                            "type": "Anything",
+                        }
+                    ],
+                }
+            ],
+        }
+        result = api.shotgun._translate_filters(filters, "all")
+        self.assertEqual(result, expected)
+
+    def test_related_object_entity_optimization(self):
+        filters = [
+            [
+                "project",
+                "is",
+                {"foo": "foo", "bar": "bar", "id": 999, "baz": "baz", "type": "Anything"},
+            ],
+        ]
+        expected = {
+            "logical_operator": "and",
+            "conditions": [
+                {
+                    "path": "project",
+                    "relation": "is",
+                    "values": [
+                        {
+                            "id": 999,
+                            "type": "Anything",
+                        }
+                    ],
+                }
+            ],
+        }
+        os.environ["SHOTGUN_API_ENABLE_ENTITY_OPTIMIZATION"] = "1"
+        api.Shotgun("http://server_path", "script_name", "api_key", connect=False)
+        result = api.shotgun._translate_filters(filters, "all")
+        self.assertEqual(result, expected)
+
 
 class TestCerts(unittest.TestCase):
     # A dummy bad url provided by Amazon
@@ -506,5 +563,6 @@ def _test_mimetypes_import(self, platform, major, minor, patch_number, result, m
         mock.platform = platform
         self.assertEqual(_is_mimetypes_broken(), result)
 
+
 if __name__ == '__main__':
     unittest.main()