Merge pull request #222 from descarteslabs/export/2018-05-03

v0.8.1 Release

Author: jpoehnelt

.travis.yml

@@ -9,7 +9,7 @@ before_install:
 - pip install --upgrade pip setuptools
 install:
 - pip install --upgrade -r requirements.txt
-- pip install flake8 coverage nose mock
+- pip install flake8 coverage nose mock responses shapely
 script:
 - if [[ $TRAVIS_PYTHON_VERSION == "3.5" ]]; then nosetests --with-coverage --cover-package=descarteslabs
   --with-doctest --doctest-options=+ELLIPSIS,+NORMALIZE_WHITESPACE && flake8; fi

README.md

@@ -36,6 +36,15 @@ Changelog
 =========
 
 ## [Unreleased]
+## [0.8.1] - 2018-05-03
+### Changed
+- Switched to `start_datetime` argument pattern instead of `start_date`
+- Fixed minor regression with `descarteslabs.ext` clients
+- Deprecated token param for `Service` class
+
+### Added
+- Raster stack method
+
 ## [0.8.0] - 2018-03-29
 ### Changed
 - Removed deprecated searching by `const_id`
@@ -198,7 +207,8 @@ metadata.features for iterating over large search results
 ### Added
 - Initial release of client library
 
-[Unreleased]: https://github.com/descarteslabs/descarteslabs-python/compare/v0.8.0...HEAD
+[Unreleased]: https://github.com/descarteslabs/descarteslabs-python/compare/v0.8.1...HEAD
+[0.8.1]: https://github.com/descarteslabs/descarteslabs-python/compare/v0.8.0...v0.8.1
 [0.8.0]: https://github.com/descarteslabs/descarteslabs-python/compare/v0.7.0...v0.8.0
 [0.7.0]: https://github.com/descarteslabs/descarteslabs-python/compare/v0.6.2...v0.7.0
 [0.6.2]: https://github.com/descarteslabs/descarteslabs-python/compare/v0.6.1...v0.6.2

descarteslabs/client/addons.py

@@ -38,3 +38,13 @@ def __call__(self, *args, **kwargs):
     import blosc
 except ImportError:
     blosc = ThirdParty("blosc")
+
+try:
+    import shapely.geometry
+except ImportError:
+    shapely = ThirdParty("shapely")
+
+try:
+    import concurrent.futures
+except ImportError:
+    concurrent = ThirdParty("futures")

descarteslabs/client/auth/auth.py

@@ -12,20 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import requests
-from urllib3.util.retry import Retry
-from requests.adapters import HTTPAdapter
-
 import base64
 import datetime
 import errno
 import json
 import os
 import random
-import six
 import stat
-from hashlib import sha1
 import warnings
+from hashlib import sha1
+
+import requests
+import six
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 
 from descarteslabs.client.exceptions import AuthError, OauthError
 
@@ -58,9 +58,9 @@ def makedirs_if_not_exists(path):
 
 
 class Auth:
-    def __init__(self, domain="https://iam.descarteslabs.com",
+    def __init__(self, domain="https://accounts.descarteslabs.com",
                  scope=None, leeway=500, token_info_path=DEFAULT_TOKEN_INFO_PATH,
-                 client_id=None, client_secret=None, jwt_token=None):
+                 client_id=None, client_secret=None, jwt_token=None, refresh_token=None):
         """
         Helps retrieve JWT from a client id and refresh token for cli usage.
         :param domain: endpoint for auth0
@@ -70,6 +70,7 @@ def __init__(self, domain="https://iam.descarteslabs.com",
         :param client_id: JWT client id
         :param client_secret: JWT client secret
         :param jwt_token: the JWT token, if we already have one
+        :param refresh_token: the refresh token
         """
         self.token_info_path = token_info_path
 
@@ -84,26 +85,26 @@ def __init__(self, domain="https://iam.descarteslabs.com",
         self.client_id = client_id if client_id else os.environ.get('CLIENT_ID', token_info.get('client_id', None))
         self.client_secret = client_secret if client_secret else os.environ.get('CLIENT_SECRET', token_info.get(
             'client_secret', None))
+        self.refresh_token = refresh_token if refresh_token \
+            else os.environ.get('DESCARTESLABS_REFRESH_TOKEN', token_info.get('refresh_token', None))
+        self.scope = scope if scope else token_info.get('scope')
         self._token = jwt_token if jwt_token else os.environ.get('JWT_TOKEN', token_info.get('jwt_token', None))
 
         if token_info:
             # If the token was read from a path but environment variables were set, we may need
             # to reset the token.
             client_id_changed = token_info.get('client_id', None) != self.client_id
             client_secret_changed = token_info.get('client_secret', None) != self.client_secret
+            refresh_token_changed = token_info.get('refresh_token', None) != self.refresh_token
 
-            if client_id_changed or client_secret_changed:
+            if client_id_changed or client_secret_changed or refresh_token_changed:
                 self._token = None
 
         self._namespace = None
-
+        self._session = None
         self.domain = domain
-        self.scope = scope
         self.leeway = leeway
 
-        if self.scope is None:
-            self.scope = ['openid', 'name', 'groups']
-
     @classmethod
     def from_environment_or_token_json(cls, **kwargs):
         """
@@ -120,18 +121,18 @@ def from_environment_or_token_json(cls, **kwargs):
     def token(self):
         if self._token is None:
             self._get_token()
-
-        exp = self.payload.get('exp')
-
-        if exp is not None:
-            now = (datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)).total_seconds()
-            if now + self.leeway > exp:
-                try:
-                    self._get_token()
-                except AuthError as e:
-                    # Unable to refresh, raise if now > exp
-                    if now > exp:
-                        raise e
+        else:  # might have token but could be close to expiration
+            exp = self.payload.get('exp')
+
+            if exp is not None:
+                now = (datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)).total_seconds()
+                if now + self.leeway > exp:
+                    try:
+                        self._get_token()
+                    except AuthError as e:
+                        # Unable to refresh, raise if now > exp
+                        if now > exp:
+                            raise e
 
         return self._token
 
@@ -148,38 +149,65 @@ def payload(self):
         claims = token.split(b'.')[1]
         return json.loads(base64url_decode(claims).decode('utf-8'))
 
+    @property
+    def session(self):
+        if self._session is None:
+            self._session = requests.Session()
+            retries = Retry(total=5,
+                            backoff_factor=random.uniform(1, 10),
+                            method_whitelist=frozenset(['GET', 'POST']),
+                            status_forcelist=[429, 500, 502, 503, 504])
+
+            self._session.mount('https://', HTTPAdapter(max_retries=retries))
+
+        return self._session
+
     def _get_token(self, timeout=100):
         if self.client_id is None:
-            raise AuthError("Could not find CLIENT_ID")
-
-        if self.client_secret is None:
-            raise AuthError("Could not find CLIENT_SECRET")
-
-        s = requests.Session()
-        retries = Retry(total=5,
-                        backoff_factor=random.uniform(1, 10),
-                        method_whitelist=frozenset(['GET', 'POST']),
-                        status_forcelist=[429, 500, 502, 503, 504])
-
-        s.mount('https://', HTTPAdapter(max_retries=retries))
-
-        headers = {"content-type": "application/json"}
-        params = {
-            "scope": " ".join(self.scope),
-            "client_id": self.client_id,
-            "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
-            "target": self.client_id,
-            "api_type": "app",
-            "refresh_token": self.client_secret
-        }
-        r = s.post(self.domain + "/auth/delegation", headers=headers, data=json.dumps(params), timeout=timeout)
+            raise AuthError("Could not find client_id")
+
+        if self.client_secret is None and self.refresh_token is None:
+            raise AuthError("Could not find client_secret or refresh token")
+
+        if self.client_id in ["ZOBAi4UROl5gKZIpxxlwOEfx8KpqXf2c"]:  # TODO(justin) remove legacy handling
+            # TODO (justin) insert deprecation warning
+            if self.scope is None:
+                scope = ['openid', 'name', 'groups']
+            else:
+                scope = self.scope
+            params = {
+                "scope": " ".join(scope),
+                "client_id": self.client_id,
+                "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
+                "target": self.client_id,
+                "api_type": "app",
+                "refresh_token": self.refresh_token if self.refresh_token is not None else self.client_secret
+            }
+        else:
+            params = {
+                "client_id": self.client_id,
+                "grant_type": "refresh_token",
+                "refresh_token": self.refresh_token if self.refresh_token is not None else self.client_secret
+            }
+
+            if self.scope is not None:
+                params["scope"] = " ".join(self.scope)
+
+        r = self.session.post(self.domain + "/token", json=params, timeout=timeout)
 
         if r.status_code != 200:
             raise OauthError("%s: %s" % (r.status_code, r.text))
 
         data = r.json()
-        self._token = data['id_token']
+        access_token = data.get('access_token')
+        id_token = data.get('id_token')  # TODO(justin) remove legacy id_token usage
 
+        if access_token is not None:
+            self._token = access_token
+        elif id_token is not None:
+            self._token = id_token
+        else:
+            raise OauthError("could not retrieve token")
         token_info = {}
 
         if self.token_info_path:

descarteslabs/client/auth/cli.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
 
 import os
 import stat
@@ -39,6 +38,8 @@ def auth_handler(args):
         if s:
 
             token_info = json.loads(base64url_decode(s).decode('utf-8'))
+            if "refresh_token" not in token_info:  # TODO(justin) legacy for previous IDP
+                token_info['refresh_token'] = token_info.get("client_secret")
 
             token_info_directory = os.path.dirname(DEFAULT_TOKEN_INFO_PATH)
             makedirs_if_not_exists(token_info_directory)
@@ -76,6 +77,7 @@ def auth_handler(args):
         auth.token
         print('%s=%s' % ('CLIENT_ID', auth.client_id))
         print('%s=%s' % ('CLIENT_SECRET', auth.client_secret))
+        print('%s=%s' % ('REFRESH_TOKEN', auth.refresh_token))
 
     if args.command == 'version':
         print(__version__)

descarteslabs/client/auth/tests/test_auth.py

@@ -12,36 +12,90 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
+import six
 import unittest
+
+import responses
+from mock import patch
+
 from descarteslabs.client.auth import Auth
-import requests
-import json
 
 
-# flake8: noqa
-anon_token = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJncm91cHMiOlsicHVibGljIl0sImlzcyI6Imh0dHBzOi8vZGVzY2FydGVzbGFicy5hdXRoMC5jb20vIiwic3ViIjoiZGVzY2FydGVzfGFub24tdG9rZW4iLCJhdWQiOiJaT0JBaTRVUk9sNWdLWklweHhsd09FZng4S3BxWGYyYyIsImV4cCI6OTk5OTk5OTk5OSwiaWF0IjoxNDc4MjAxNDE5fQ.QL9zq5SkpO7skIy0niIxI0B92uOzZT5t1abuiJaspRI"
+def token_response_callback(request):
+    body = request.body
+    if not isinstance(body, six.text_type):
+        body = body.decode('utf-8')
+
+    data = json.loads(body)
+
+    required_fields = ['client_id', 'grant_type', 'refresh_token']
+    legacy_required_fields = ["api_type", "target"]
+
+    if not all(field in data for field in required_fields):
+        return 400, {"Content-Type": "application/json"}, json.dumps("missing fields")
+
+    if data['grant_type'] == "urn:ietf:params:oauth:grant-type:jwt-bearer" \
+            and all(field in data for field in legacy_required_fields):
+        return 200, {"Content-Type": "application/json"}, json.dumps(dict(id_token="id_token"))
+
+    if data['grant_type'] == "refresh_token" \
+            and all(field not in data for field in legacy_required_fields):
+        return 200, {"Content-Type": "application/json"}, json.dumps(dict(access_token="access_token",
+                                                                          id_token="id_token"))
+    return 400, {"Content-Type": "application/json"}, json.dumps(data)
 
 
 class TestAuth(unittest.TestCase):
+    @responses.activate
     def test_get_token(self):
-        # get a jwt
-        auth = Auth.from_environment_or_token_json()
-        self.assertIsNotNone(auth.token)
+        responses.add(responses.POST, 'https://accounts.descarteslabs.com/token',
+                      json=dict(access_token="access_token"),
+                      status=200)
+        auth = Auth(token_info_path=None, client_secret="client_secret", client_id="client_id")
+        auth._get_token()
+
+        self.assertEqual("access_token", auth._token)
 
-        # validate the jwt
-        url = "https://descarteslabs.auth0.com" + "/tokeninfo"
-        params = {"id_token": auth.token}
-        headers = {"content-type": "application/json"}
-        r = requests.post(url, data=json.dumps(params), headers=headers)
-        self.assertEqual(200, r.status_code)
+    @responses.activate
+    def test_get_token_legacy(self):
+        responses.add(responses.POST, 'https://accounts.descarteslabs.com/token',
+                      json=dict(id_token="id_token"), status=200)
+        auth = Auth(token_info_path=None, client_secret="client_secret", client_id="client_id")
+        auth._get_token()
 
+        self.assertEqual("id_token", auth._token)
+
+    @patch("descarteslabs.client.auth.Auth.payload", new=dict(sub="asdf"))
     def test_get_namespace(self):
-        auth = Auth.from_environment_or_token_json()
-        self.assertIsNotNone(auth.namespace)
+        auth = Auth(token_info_path=None, client_secret="client_secret", client_id="client_id")
+        self.assertEqual(auth.namespace, "3da541559918a808c2402bba5012f6c60b27661c")
 
     def test_init_token_no_path(self):
-        auth = Auth(jwt_token=anon_token, token_info_path=None, client_id="foo")
-        self.assertEquals(anon_token, auth._token)
+        auth = Auth(jwt_token="token", token_info_path=None, client_id="foo")
+        self.assertEqual("token", auth._token)
+
+    @responses.activate
+    def test_get_token_schema_internal_only(self):
+        responses.add_callback(responses.POST, 'https://accounts.descarteslabs.com/token',
+                               callback=token_response_callback)
+        auth = Auth(token_info_path=None, refresh_token="refresh_token", client_id="client_id")
+        auth._get_token()
+
+        self.assertEqual("access_token", auth._token)
+
+        auth = Auth(token_info_path=None, client_secret="refresh_token", client_id="client_id")
+        auth._get_token()
+
+        self.assertEqual("access_token", auth._token)
+
+    @responses.activate
+    def test_get_token_schema_legacy_internal_only(self):
+        responses.add_callback(responses.POST, 'https://accounts.descarteslabs.com/token',
+                               callback=token_response_callback)
+        auth = Auth(token_info_path=None, client_secret="client_secret", client_id="ZOBAi4UROl5gKZIpxxlwOEfx8KpqXf2c")
+        auth._get_token()
+        self.assertEqual("id_token", auth._token)
 
 
 if __name__ == '__main__':

descarteslabs/client/scripts/cli.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 # flake8: noqa
-from __future__ import print_function
+
 
 import argparse
 from descarteslabs.client.auth.cli import auth_handler