SG-11735: Support for Python 3! (#202)

Python 3 support is here! This release is also compatible with Python 2.6 and 2.7, as it was before. This release includes a large number of changes to accommodate the newer versions of Python, but functionally it should remain the same.

.coveragerc

@@ -16,5 +16,5 @@
 source=shotgun_api3
 omit=
     shotgun_api3/lib/httplib2/*
-    shotgun_api3/lib/simplejson/*
+    shotgun_api3/lib/six.py
     shotgun_api3/lib/xmlrpclib.py

.flake8

@@ -0,0 +1,13 @@
+# Copyright (c) 2019 Shotgun Software Inc.
+#
+# CONFIDENTIAL AND PROPRIETARY
+#
+# This work is provided "AS IS" and subject to the Shotgun Pipeline Toolkit
+# Source Code License included in this distribution package. See LICENSE.
+# By accessing, using, copying or modifying this work you indicate your
+# agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
+# not expressly granted therein are reserved by Shotgun Software Inc.
+
+[flake8]
+max-line-length = 120
+exclude = shotgun_api3/lib/httplib2/*,shotgun_api3/lib/six.py,shotgun_api3/lib/xmlrpclib.py,tests/httplib2test.py,tests/mock.py

.gitignore

@@ -1,3 +1,13 @@
+# Copyright (c) 2019 Shotgun Software Inc.
+#
+# CONFIDENTIAL AND PROPRIETARY
+#
+# This work is provided "AS IS" and subject to the Shotgun Pipeline Toolkit
+# Source Code License included in this distribution package. See LICENSE.
+# By accessing, using, copying or modifying this work you indicate your
+# agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
+# not expressly granted therein are reserved by Shotgun Software Inc.
+
 #python specific
 *.pyc
 

.travis.yml

@@ -1,22 +1,89 @@
+# Copyright (c) 2019 Shotgun Software Inc.
+#
+# CONFIDENTIAL AND PROPRIETARY
+#
+# This work is provided "AS IS" and subject to the Shotgun Pipeline Toolkit
+# Source Code License included in this distribution package. See LICENSE.
+# By accessing, using, copying or modifying this work you indicate your
+# agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
+# not expressly granted therein are reserved by Shotgun Software Inc.
+
 language: python
-python:
-  - "2.6"
-  - "2.7"
 
-# use trusty dist, since xenial (default) does not support python 2.6
-dist: trusty
+# We need to test on multiple distros, since python 2.6 is not supported in xenial
+# and 3.7 is not supported in trusty.  For now just use trusty to test 2.6, and use
+# the more modern xenial for 2.7 and 3.7.
+
+# Additionally, we'll test on multiple sites.  To accomplish this, we store the
+# secrets for the second site in a set of additional environment variables,
+# and copy those into the standard environment variables when running on the
+# second site.
+matrix:
+  include:
+
+    # Use trusty dist for python 2.6, since the built-in xenial distro (default)
+    # does not support Python 2.6
+    # See https://docs.travis-ci.com/user/reference/xenial/#python-support
+
+    # Python 2.6, Server #1
+    - dist: trusty
+      python: "2.6"
+      # flake8 does not support python 2.6, so only run it on 2.7+
+      env:
+        - RUN_FLAKE=false
+
+    # Python 2.6, Server #2
+    - dist: trusty
+      python: "2.6"
+      # flake8 does not support python 2.6, so only run it on 2.7+
+      env:
+        - RUN_FLAKE=false
+        - SG_SERVER_URL=$SG_SERVER_URL_2
+        - SG_API_KEY=$SG_API_KEY_2
+        - SG_HUMAN_PASSWORD=$SG_HUMAN_PASSWORD_2
+
+    # Test python 2.7 and 3.7 on Xenial.
+
+    # Python 2.7, Server #1
+    - dist: xenial
+      python: "2.7"
+      env:
+        - RUN_FLAKE=true
+
+    # Python 2.7, Server #2
+    - dist: xenial
+      python: "2.7"
+      env:
+        - RUN_FLAKE=true
+        - SG_SERVER_URL=$SG_SERVER_URL_2
+        - SG_API_KEY=$SG_API_KEY_2
+        - SG_HUMAN_PASSWORD=$SG_HUMAN_PASSWORD_2
+
+    # Python 3.7, Server #1
+    - dist: xenial
+      python: "3.7"
+      env:
+        - RUN_FLAKE=true
+
+    # Python 3.7, Server #1
+    - dist: xenial
+      python: "3.7"
+      env:
+        - RUN_FLAKE=true
+        - SG_SERVER_URL=$SG_SERVER_URL_2
+        - SG_API_KEY=$SG_API_KEY_2
+        - SG_HUMAN_PASSWORD=$SG_HUMAN_PASSWORD_2
 
 # command to install dependencies
 install:
   - pip install -r tests/ci_requirements.txt
 before_script:
   - cp ./tests/example_config ./tests/config
-# command to run tests
-script: coverage run -m nose
+# Run flake and then run tests
+script:
+  - if [ -n "$RUN_FLAKE" = true ]; then flake8; fi
+  - coverage run -m nose --config="nose.cfg"
 after_success: coveralls
 notifications:
   email:
     - [email protected]
-  slack:
-    rooms:
-      secure: NgcVxptQji2OL7EKx5owh21tOl4rRg51ydYhLPHpBIpBVzlqPxi1jbMHpcbl8mG+GuhU6y1D6kVQgQfuAfIWx5YOc4FodhOFZgeS+eol5RTmEepl1OT6XmAVgEclSZUYQmK25OkcxnFnOV/31RuDPXWcgCvSBZqTy6lAbOZ5VwQ=

HISTORY.rst

@@ -4,6 +4,10 @@ Shotgun Python API Changelog
 
 Here you can see the full list of changes between each Python API release.
 
+v3.1.0 (2019 July 29)
+=====================
+- Adds support for Python 3.7
+
 v3.0.41 (2019 June 28)
 =====================
 - Adds an optional sleep between retries specified via the `SHOTGUN_API_RETRY_INTERVAL` environment variable, or by setting `sg.config.rpc_attempt_interval`.

README.md

@@ -1,10 +1,10 @@
-[![VFX Platform](https://img.shields.io/badge/vfxplatform-2018-yellow.svg)](http://www.vfxplatform.com/)
+[![VFX Platform](https://img.shields.io/badge/vfxplatform-2020-blue.svg)](http://www.vfxplatform.com/)
+[![Python 2.6 2.7 3.7](https://img.shields.io/badge/python-2.6%20%7C%202.7%20%7C%203.7-blue.svg)](https://www.python.org/)
 [![Reference Documentation](http://img.shields.io/badge/doc-reference-blue.svg)](http://developer.shotgunsoftware.com/python-api)
 [![Build Status Linux](https://secure.travis-ci.org/shotgunsoftware/python-api.svg?branch=master)](http://travis-ci.org/shotgunsoftware/python-api)
 [![Build status Windows](https://ci.appveyor.com/api/projects/status/slvw7u4jatvdly98/branch/master?svg=true
 )](https://ci.appveyor.com/project/jfboismenu/python-api/branch/master)
 [![Coverage Status](https://coveralls.io/repos/github/shotgunsoftware/python-api/badge.svg?branch=master)](https://coveralls.io/github/shotgunsoftware/python-api?branch=master)
-[![Linting](https://img.shields.io/badge/PEP8%20by-Hound%20CI-a873d1.svg)](https://houndci.com)
 
 # Shotgun Python API
 
@@ -15,14 +15,10 @@ The latest version can always be found at http://github.com/shotgunsoftware/pyth
 ## Minimum Requirements
 
 * Shotgun server v2.4.12+.
-* Python v2.6 - v2.7.
-
-## High Performance Requirements
-
-* Install [simplejson 2.1.6](http://pypi.python.org/pypi/simplejson/2.1.6)
+* Python v2.6 - v2.7 or v3.7
 
 ## Documentation
-Tutorials and detailed documentation about the Python API are available at http://developer.shotgunsoftware.com/python-api). 
+Tutorials and detailed documentation about the Python API are available at http://developer.shotgunsoftware.com/python-api).
 
 Some useful direct links:
 
@@ -39,12 +35,70 @@ You can see the [full history of the Python API on the documentation site](http:
 ## Updating HTTPLib2
 
 1. Download the latest version of HTTPLib2 at https://pypi.org/project/httplib2.
-2. Extract the python2/httplib2 into shotgun_api3/lib/http2lib without the test folder.
-3. Scan the files for any references to importing httplib2 and make sure they import "from ." instead of "from httplib2" because the library isn't in the Python path.
+2. Extract the python2/httplib2 into shotgun_api3/lib/http2lib/python2 without the test folder.
+3. Extract the python3/httplib2 into shotgun_api3/lib/http2lib/python3 without the test folder.
+4. Scan the files for any references to importing httplib2 and make sure they import "from ." instead of "from httplib2" because the library isn't in the Python path.
+
+## Maintaining Python 2 and 3 compatibility
+
+python-api should remain compatible with both Python 2, and 3.  To make this easier, we use [six](https://six.readthedocs.io/).  When adding code that works with types that have changed between Python 2 and 3, notably strings and files, it's advisable to use the `six` types for casting and comparisons. Be sure to follow Python 2 and 3 compatible conventions in code, especially when raising or capturing exceptions and printing. While we don't use `future`, [this page](https://python-future.org/compatible_idioms.html) contains a fairly comprehensive list of Python 2/3 compatibility sticking points to look out for.
+
+Additionally, the [python-modernize](https://python-modernize.readthedocs.io/en/latest/) tool can be helpful when updating Python 2 code for Python 3 compatibility.
+
+### Examples:
+
+#### Comparisons against changed types:
+
+Python 2:
+
+```
+if isinstance(my_variable, str):
+```
+
+Python 2/3:
+
+```
+if isinstance(my_variable, six.string_types):
+```
+
+#### Catching exceptions
+
+Python 2:
+
+```
+except SomeExceptionType, e:
+    print "I like to swallow exceptions!"
+```
+
+Python 2/3:
+
+```
+from __future__ import print_function
+except SomeExceptionType as e:
+    print("I like to swallow exceptions!")
+```
+
+#### Print statements
+
+Python 2:
+
+```
+print "My spoon is too big!"
+```
+
+Python 2/3:
+
+```
+from __future__ import print_function
+print("My spoon is too big!")
+```
+
+
+Additionally, when testing locally, tests should be run for both python 2 and python 3 to ensure changes won't break cross-compatibility.
 
-## Tests 
+## Tests
 
-Integration and unit tests are provided. 
+Integration and unit tests are provided.
 
 - All tests require the [nose unit testing tools](http://nose.readthedocs.org), and a `tests/config` file (you can copy an example from `tests/example_config`).
 - Tests can be run individually like this: `nosetest tests/test_client.py`
@@ -58,11 +112,11 @@ Integration and unit tests are provided.
 
 1) Update the Changelog in the `HISTORY.rst` file
     - Add bullet points for any changes that have happened since the previous release. This may include changes you did not make so look at the commit history and make sure we don't miss anything. If you notice something was done that wasn't added to the changelog, hunt down that engineer and make them feel guilty for not doing so. This is a required step in making changes to the API.
-    - Try and match the language of previous change log messages. We want to keep a consistent voice. 
+    - Try and match the language of previous change log messages. We want to keep a consistent voice.
     - Make sure the date of the release matches today. We try and keep this TBD until we're ready to do a release so it's easy to catch that it needs to be updated.
     - Make sure the version number is filled out and correct. We follow semantic versioning. Or more correctly, we should be following it.
 2) Ensure any changes or additions to public methods are documented
-    - Update the Github wiki, and usually you'll need to update the Method Reference page with concise and exact documentation of the changes that are in this release. 
+    - Update the Github wiki, and usually you'll need to update the Method Reference page with concise and exact documentation of the changes that are in this release.
     - Ensure that doc strings are updated in the code itself to work with Sphinx and are correctly formatted.
     - Examples are always good especially if this a new feature or method.
     - Think about a new user to the API trying to figure out how to use the features you're documenting.
@@ -76,7 +130,7 @@ Integration and unit tests are provided.
     - Add more detailed information regarding the changes in this release. This is a great place to add examples, and reasons for the change!
 
 ### Letting the world know
-We usually send an email to the `shotgun-dev` list with an announcement of the release and highlight the changes. 
+We usually send an email to the `shotgun-dev` list with an announcement of the release and highlight the changes.
 
 ### Prepare for the Next Dev Cycle
 1) Update the `__version__` value in `shotgun_api3/shotgun.py` to the next version number with `.dev` appended to it. For example, `v3.0.24.dev`

appveyor.yml

@@ -12,7 +12,9 @@
 
 environment:
   matrix:
+    - PYTHON: "C:\\Python26"
     - PYTHON: "C:\\Python27"
+    - PYTHON: "C:\\Python37"
 
   # To update these values, visit AppVeyor's site, click the user icon and scroll down to Encrypt Data.
   SG_SERVER_URL:
@@ -28,6 +30,16 @@ environment:
 
 build: off
 
+before_test:
+  # The certificate store of the build machine is out of date and is missing a
+  # new certificate authority used by Amazon for S3, so we need to update it.
+  # See https://developer.shotgunsoftware.com/c593f0aa/
+  - ps: |
+      $cert_url = "https://www.amazontrust.com/repository/SFSRootCAG2.cer"
+      $cert_file = New-TemporaryFile
+      Invoke-WebRequest -Uri $cert_url -UseBasicParsing -OutFile $cert_file.FullName
+      Import-Certificate -FilePath $cert_file.FullName -CertStoreLocation Cert:\LocalMachine\Root
+
 test_script:
   # Put your test command here.
   # If you don't need to build C extensions on 64-bit Python 3.3 or 3.4,

docs/cookbook/examples/ami_version_packager.rst

@@ -253,5 +253,5 @@ It is intended to be used in conjunction with the script dicussed in :ref:`ami_h
             raise ShotgunException("Unknown action... :%s" % sa.action)
             
         
-        print "\nVersion Packager done!"
+        print("\nVersion Packager done!")
 

docs/cookbook/examples/basic_create_shot.rst

@@ -89,7 +89,7 @@ The Complete Example
         }
         result = sg.create('Shot', data)    
         pprint(result)
-        print "The id of the %s is %d." % (result['type'], result['id'])
+        print("The id of the {} is {}.".format(result['type'], result['id']))
 
 And here is the output::
 

docs/cookbook/examples/svn_integration.rst

@@ -117,11 +117,11 @@ link to the Revision.
                            'created_by':{"type":"HumanUser", "id":result['id']}
                            }
            revision = sg.create("Revision", parameters)
-           print "created Revision #"+str(revision_code)
+           print("created Revision #"+str(revision_code))
        
        # Send error message if valid HumanUser is not found
        else:
-           print "Unable to find valid Shotgun User with login: "+author+", Revision not created in Shotgun."
+           print("Unable to find valid Shotgun User with login: "+author+", Revision not created in Shotgun.")
 
 
 

docs/installation.rst

@@ -9,7 +9,7 @@ Minimum Requirements
 - Shotgun server v5.4.14 or higher for API v3.0.16+.
 - Shotgun server v2.4.12 or higher for API v3.0.8.
 - Shotgun server v1.8 or higher for API v3.0.7.
-- Python v2.4 - v2.7. (We do not currently support Python 3)
+- Python 2.6 - 2.7 or Python 3.7
 
 .. note::
     Some features of the API are only supported by more recent versions of the Shotgun server.

nose.cfg

@@ -0,0 +1,12 @@
+# Copyright (c) 2019 Shotgun Software Inc.
+#
+# CONFIDENTIAL AND PROPRIETARY
+#
+# This work is provided "AS IS" and subject to the Shotgun Pipeline Toolkit
+# Source Code License included in this distribution package. See LICENSE.
+# By accessing, using, copying or modifying this work you indicate your
+# agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
+# not expressly granted therein are reserved by Shotgun Software Inc.
+
+[nosetests]
+exclude-dir=shotgun_api3/lib

run-tests

@@ -1 +1,11 @@
-clear && find ./ -name ".coverage" -delete && find ./ -name "*.pyc" -delete && nosetests -vd  --with-cover --cover-package=shotgun_api3
+# Copyright (c) 2019 Shotgun Software Inc.
+#
+# CONFIDENTIAL AND PROPRIETARY
+#
+# This work is provided "AS IS" and subject to the Shotgun Pipeline Toolkit
+# Source Code License included in this distribution package. See LICENSE.
+# By accessing, using, copying or modifying this work you indicate your
+# agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
+# not expressly granted therein are reserved by Shotgun Software Inc.
+
+clear && find ./ -name ".coverage" -delete && find ./ -name "*.pyc" -delete && nosetests -vd --config="nose.cfg" --with-cover --cover-package=shotgun_api3

setup.py

@@ -1,4 +1,14 @@
 # -*- coding: utf-8 -*-
+# Copyright (c) 2019 Shotgun Software Inc.
+#
+# CONFIDENTIAL AND PROPRIETARY
+#
+# This work is provided "AS IS" and subject to the Shotgun Pipeline Toolkit
+# Source Code License included in this distribution package. See LICENSE.
+# By accessing, using, copying or modifying this work you indicate your
+# agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
+# not expressly granted therein are reserved by Shotgun Software Inc.
+
 import sys
 from setuptools import setup, find_packages
 
@@ -17,7 +27,7 @@
 
 setup(
     name='shotgun_api3',
-    version='3.0.41',
+    version='3.1.0',
     description='Shotgun Python API ',
     long_description=readme,
     author='Shotgun Software',