From 5b68d176232e46de4ee9bd2743d155014d92ffde Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Thu, 8 Aug 2019 10:53:18 -0700 Subject: [PATCH 01/17] Upgrade coverage to 4.x --- .travis.yml | 2 ++ Dockerfile | 3 ++- confidant/routes/v1.py | 2 +- confidant/scripts/bootstrap.py | 4 ++-- requirements.in | 9 ++------- requirements.txt | 12 ++++++++---- requirements3.txt | 9 +++++---- 7 files changed, 22 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index c644994b..e33375ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,6 @@ +dist: xenial language: python +python: 2.7 env: - REPO=lyft/confidant sudo: required diff --git a/Dockerfile b/Dockerfile index 18c87874..41ce8dae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,8 @@ RUN virtualenv /venv && \ pip install -r piptools_requirements.txt && \ pip install -r requirements.txt -RUN gem install rb-inotify -v 0.9.10 && \ +RUN gem install ffi -v 1.10.0 && \ + gem install rb-inotify -v 0.9.10 && \ gem install compass -v 1.0.3 && \ npm install grunt-cli && \ npm install diff --git a/confidant/routes/v1.py b/confidant/routes/v1.py index 7b160f70..2de461e8 100644 --- a/confidant/routes/v1.py +++ b/confidant/routes/v1.py @@ -1213,7 +1213,7 @@ def update_blind_credential(id): def generate_value(): value = kms_client.generate_random(NumberOfBytes=128)['Plaintext'] value = base64.urlsafe_b64encode(value) - value = re.sub('[\W_]+', '', value) + value = re.sub(r'[\W_]+', '', value) if len(value) > VALUE_LENGTH: value = value[:VALUE_LENGTH] return jsonify({'value': value}) diff --git a/confidant/scripts/bootstrap.py b/confidant/scripts/bootstrap.py index c490ac29..9c36003f 100644 --- a/confidant/scripts/bootstrap.py +++ b/confidant/scripts/bootstrap.py @@ -41,7 +41,7 @@ def run(self, _in, _out): } data = json.dumps(data) if _out == '-': - print data + print(data) else: with open(os.path.join(_out), 'w') as f: f.write(data) @@ -57,7 +57,7 @@ def run(self, _out): data = settings.encrypted_settings.get_all_secrets() data = yaml.safe_dump(data, default_flow_style=False, indent=2) if _out == '-': - print data + print(data) else: with open(os.path.join(_out), 'w') as f: f.write(data) diff --git a/requirements.in b/requirements.in index 9ce90d14..7ef54b5c 100644 --- a/requirements.in +++ b/requirements.in @@ -109,17 +109,12 @@ guard==1.0.1 # The modular source code checker: pep8, pyflakes and co # License: MIT # Upstream url: http://bitbucket.org/tarek/flake8 -flake8==2.3.0 +flake8>3 # Measures code coverage and emits coverage reports # Licence: BSD # Upstream url: https://pypi.python.org/pypi/coverage -coverage==3.7.1 - -# tool to check your Python code against some of the style conventions -# License: Expat License -# Upstream url: https://github.com/jcrocholl/pep8.git -pep8==1.5.7 +coverage>4 # nose makes testing easier # License: GNU Library or Lesser General Public License (LGPL) diff --git a/requirements.txt b/requirements.txt index ad9f4fda..22eaa76f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,17 +10,20 @@ boto3==1.4.0 botocore==1.4.58 certifi==2018.4.16 cffi==1.10.0 -coverage==3.7.1 +configparser==3.7.4 # via entrypoints, flake8 +coverage==4.5.4 cryptography==2.3 defusedxml==0.5.0 # via python3-saml docutils==0.13.1 # via botocore +entrypoints==0.3 # via flake8 enum34==1.1.6 -flake8==2.3.0 +flake8==3.7.8 flask-script==2.0.5 flask-session==0.2.1 flask-sslify==0.1.5 flask==0.10.1 funcsigs==1.0.2 # via mock +functools32==3.2.3.post2 # via flake8 futures==3.0.5 # via s3transfer gevent==1.2.1 greenlet==0.4.12 @@ -41,11 +44,11 @@ ndg-httpsclient==0.4.2 nose-pathmunge==0.1.2 nose==1.3.3 pbr==2.0.0 # via mock -pep8==1.5.7 pkgconfig==1.3.1 # via xmlsec pyasn1==0.4.4 +pycodestyle==2.5.0 # via flake8 pycparser==2.18 -pyflakes==1.5.0 # via flake8 +pyflakes==2.1.1 # via flake8 pynamodb==3.2.1 pyopenssl==16.2.0 python-dateutil==2.6.0 # via botocore, pynamodb @@ -56,6 +59,7 @@ requests==2.11.1 s3transfer==0.1.10 # via boto3 six==1.10.0 statsd==3.2.1 +typing==3.7.4 # via flake8 werkzeug==0.12.1 # via flask xmlsec==1.3.3 # via python3-saml diff --git a/requirements3.txt b/requirements3.txt index 29183b6f..4c29ea11 100644 --- a/requirements3.txt +++ b/requirements3.txt @@ -9,11 +9,12 @@ authomatic==0.1.0.post1 boto3==1.4.0 botocore==1.4.58 cffi==1.10.0 -coverage==3.7.1 +coverage==4.5.4 cryptography==2.3 defusedxml==0.5.0 # via python3-saml docutils==0.13.1 # via botocore -flake8==2.3.0 +entrypoints==0.3 # via flake8 +flake8==3.7.8 flask-script==2.0.5 flask-session==0.2.1 flask-sslify==0.1.5 @@ -36,11 +37,11 @@ ndg-httpsclient==0.4.2 nose-pathmunge==0.1.2 nose==1.3.3 pbr==2.0.0 # via mock -pep8==1.5.7 pkgconfig==1.3.1 # via xmlsec pyasn1==0.4.4 +pycodestyle==2.5.0 # via flake8 pycparser==2.18 -pyflakes==1.5.0 # via flake8 +pyflakes==2.1.1 # via flake8 pynamodb==3.2.1 pyopenssl==16.2.0 python-dateutil==2.6.0 # via botocore, pynamodb From 73c0669a7e4f5709e4c3257c4d8f98813e80eb58 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Thu, 29 Aug 2019 12:49:46 -0700 Subject: [PATCH 02/17] Convert confidant to python3 --- .travis.yml | 1 - Dockerfile | 16 +++---- confidant/authnz/__init__.py | 8 ++-- confidant/authnz/userauth.py | 10 ++--- confidant/ciphermanager.py | 2 +- confidant/keymanager.py | 2 +- confidant/lib/cryptolib.py | 4 +- confidant/routes/v1.py | 14 +++---- confidant/settings.py | 2 +- confidant/utils/misc.py | 2 +- piptools_requirements.in | 36 ---------------- piptools_requirements.txt | 18 ++++---- piptools_requirements3.txt | 9 ---- requirements.in | 18 ++++---- requirements.txt | 29 ++++++------- requirements3.txt | 28 ++++++------- .../confidant/authnz/authnz_test.py | 42 ++++++------------- tests/unit/confidant/ciphermanager_test.py | 2 +- tests/unit/confidant/keymanager_test.py | 2 +- 19 files changed, 90 insertions(+), 155 deletions(-) delete mode 100644 piptools_requirements.in diff --git a/.travis.yml b/.travis.yml index e33375ab..cc0059b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ dist: xenial language: python -python: 2.7 env: - REPO=lyft/confidant sudo: required diff --git a/Dockerfile b/Dockerfile index 41ce8dae..f11552b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:trusty +FROM ubuntu:bionic LABEL maintainer="rlane@lyft.com" RUN apt-get update \ @@ -11,22 +11,18 @@ RUN apt-get update \ make ruby-dev nodejs git-core \ # For backend gcc pkg-config \ - python-dev python-virtualenv \ + python3-dev virtualenv \ libffi-dev libxml2-dev libxmlsec1-dev \ && apt-get clean && rm -rf /var/lib/apt/lists/* -COPY ./piptools_requirements.txt /srv/confidant/piptools_requirements.txt -COPY ./requirements.txt /srv/confidant/requirements.txt -COPY ./package.json /srv/confidant/package.json -COPY ./bower.json /srv/confidant/bower.json +COPY package.json bower.json piptools_requirements*.txt requirements*.txt /srv/confidant/ WORKDIR /srv/confidant ENV PATH=/venv/bin:$PATH -RUN virtualenv /venv && \ - pip install --upgrade pip && \ - pip install -r piptools_requirements.txt && \ - pip install -r requirements.txt +RUN virtualenv /venv -ppython3 && \ + pip install --no-cache -r piptools_requirements3.txt && \ + pip install --no-cache -r requirements3.txt RUN gem install ffi -v 1.10.0 && \ gem install rb-inotify -v 0.9.10 && \ diff --git a/confidant/authnz/__init__.py b/confidant/authnz/__init__.py index 9c425e00..2cfb8399 100644 --- a/confidant/authnz/__init__.py +++ b/confidant/authnz/__init__.py @@ -185,7 +185,7 @@ def decorated(*args, **kwargs): logging.debug(msg) if user_type_has_privilege( kms_auth_data['user_type'], - f.func_name): + f.__name__): g.user_type = kms_auth_data['user_type'] g.auth_type = 'kms' g.account = account_for_key_alias(token_data['key_alias']) @@ -193,7 +193,7 @@ def decorated(*args, **kwargs): return f(*args, **kwargs) else: msg = '{0} is not authorized to access {1}.' - msg = msg.format(kms_auth_data['from'], f.func_name) + msg = msg.format(kms_auth_data['from'], f.__name__) logging.warning(msg) return abort(403) except keymanager.TokenDecryptionError: @@ -206,7 +206,7 @@ def decorated(*args, **kwargs): # If not using kms auth, require auth using the user_mod authn module. else: user_type = 'user' - if not user_type_has_privilege(user_type, f.func_name): + if not user_type_has_privilege(user_type, f.__name__): return abort(403) if user_mod.is_expired(): @@ -216,7 +216,7 @@ def decorated(*args, **kwargs): try: user_mod.check_authorization() except NotAuthorized as e: - logging.warning('Not authorized -- ' + e.message) + logging.warning('Not authorized -- {}'.format(e)) return abort(403) else: # User took an action, extend the expiration time. diff --git a/confidant/authnz/userauth.py b/confidant/authnz/userauth.py index df98ce52..b34c1b00 100644 --- a/confidant/authnz/userauth.py +++ b/confidant/authnz/userauth.py @@ -1,10 +1,10 @@ import abc import logging -import urlparse import datetime import random import yaml +from six.moves.urllib.parse import urlparse import flask from flask import request, session @@ -395,7 +395,7 @@ def log_in(self): if result: if result.error: msg = 'Google auth failed with error: {0}' - logging.error(msg.format(result.error.message)) + logging.error(msg.format(result.error)) return abort(403) # successful login @@ -602,7 +602,7 @@ def consume_saml_assertion(self): logging.info('SAML attributes: {!r}'.format(attributes)) # normalize attributes by flattening single-item arrays - for key, val in attributes.iteritems(): + for key, val in attributes.items(): if isinstance(val, list) and len(val) == 1: attributes[key] = val[0] @@ -618,7 +618,7 @@ def consume_saml_assertion(self): kwargs['email'] = attributes.get('email', nameid) # use first_name, last_name if present - for key, val in attributes.iteritems(): + for key, val in attributes.items(): if not getattr(key, 'lower', None): logging.error('Bad list attr {!r}'.format({key: val})) if key.lower() in ['firstname', 'first_name']: @@ -735,7 +735,7 @@ def _saml_req_dict_from_request(self, flask_request=None): if flask_request is None: flask_request = flask.request - url_data = urlparse.urlparse(flask_request.url) + url_data = urlparse(flask_request.url) if flask_request.scheme == 'https': https = 'on' diff --git a/confidant/ciphermanager.py b/confidant/ciphermanager.py index c0b4ced5..28a48656 100644 --- a/confidant/ciphermanager.py +++ b/confidant/ciphermanager.py @@ -28,7 +28,7 @@ def encrypt(self, raw): return 'DANGER_NOT_ENCRYPTED_{0}'.format(base64.b64encode(raw)) if self.version == 2: f = Fernet(self.key) - return f.encrypt(raw.encode('utf-8')) + return f.encrypt(raw.encode('utf-8')).decode('UTF-8') else: raise CipherManagerError('Bad cipher version') diff --git a/confidant/keymanager.py b/confidant/keymanager.py index 3a239c80..8c0e29ab 100644 --- a/confidant/keymanager.py +++ b/confidant/keymanager.py @@ -124,7 +124,7 @@ def decrypt_token(version, user_type, _from, token): stats.incr('token_version_{0}'.format(version)) try: token_key = '{0}{1}'.format( - hashlib.sha256(token).hexdigest(), + hashlib.sha256(token.encode()).hexdigest(), _from ) except Exception: diff --git a/confidant/lib/cryptolib.py b/confidant/lib/cryptolib.py index 1e653995..1871684d 100644 --- a/confidant/lib/cryptolib.py +++ b/confidant/lib/cryptolib.py @@ -74,7 +74,7 @@ def load_x509_certificate_pem(path): :rtype: cryptography.x509.Certificate """ - with open(path, 'r') as f: + with open(path, 'rb') as f: cert = x509.load_pem_x509_certificate(f.read(), default_backend()) return cert @@ -107,7 +107,7 @@ def _x509_certificate_bare_base64(certificate): :rtype: string """ return base64.b64encode(certificate.public_bytes( - serialization.Encoding.DER)) + serialization.Encoding.DER)).decode() def load_private_key_pem(path, password=None): diff --git a/confidant/routes/v1.py b/confidant/routes/v1.py index 2de461e8..ca42cfe8 100644 --- a/confidant/routes/v1.py +++ b/confidant/routes/v1.py @@ -62,7 +62,7 @@ def get_client_config(): 'defined': app.config['CLIENT_CONFIG'], 'generated': { 'kms_auth_manage_grants': app.config['KMS_AUTH_MANAGE_GRANTS'], - 'aws_accounts': app.config['SCOPED_AUTH_KEYS'].values(), + 'aws_accounts': list(app.config['SCOPED_AUTH_KEYS'].values()), 'xsrf_cookie_name': app.config['XSRF_COOKIE_NAME'], 'maintenance_mode': app.config['MAINTENANCE_MODE'] } @@ -278,7 +278,7 @@ def map_service_credentials(id): } return jsonify(ret), 400 - accounts = app.config['SCOPED_AUTH_KEYS'].values() + accounts = list(app.config['SCOPED_AUTH_KEYS'].values()) if data.get('account') and data['account'] not in accounts: ret = {'error': '{0} is not a valid account.'} return jsonify(ret), 400 @@ -533,7 +533,7 @@ def _pair_key_conflicts_for_credentials(credential_ids, blind_credential_ids): pair_keys[key] = [data] # Iterate the credential pair keys, if there's any keys with more than # one credential add it to the conflict dict. - for key, data in pair_keys.iteritems(): + for key, data in pair_keys.items(): if len(data) > 1: blind_ids = [k['id'] for k in data if k['data_type'] == 'blind-credential'] @@ -562,7 +562,7 @@ def _get_services_for_blind_credential(_id): def _check_credential_pair_values(credential_pairs): - for key, val in credential_pairs.iteritems(): + for key, val in credential_pairs.items(): if isinstance(val, dict) or isinstance(val, list): ret = {'error': 'credential pairs must be key: value'} return (False, ret) @@ -599,7 +599,7 @@ def _pair_key_conflicts_for_services(_id, credential_keys, services): service_map = _get_service_map(services) credential_ids = [] blind_credential_ids = [] - for credential, data in service_map.iteritems(): + for credential, data in service_map.items(): if _id == credential: continue if data['data_type'] == 'credential': @@ -636,7 +636,7 @@ def _pair_key_conflicts_for_services(_id, credential_keys, services): def _lowercase_credential_pairs(credential_pairs): - return {i.lower(): j for i, j in credential_pairs.iteritems()} + return {i.lower(): j for i, j in credential_pairs.items()} @app.route('/v1/credentials', methods=['POST']) @@ -757,7 +757,7 @@ def update_credential(id): # services conflicts = _pair_key_conflicts_for_services( id, - credential_pairs.keys(), + list(credential_pairs.keys()), services ) if conflicts: diff --git a/confidant/settings.py b/confidant/settings.py index b43f639d..17f47110 100644 --- a/confidant/settings.py +++ b/confidant/settings.py @@ -460,7 +460,7 @@ def str_env(var_name, default=''): # Configuration validation _settings_failures = False -if len(list(set(SCOPED_AUTH_KEYS.values()))) != len(SCOPED_AUTH_KEYS.values()): +if len(set(SCOPED_AUTH_KEYS.values())) != len(SCOPED_AUTH_KEYS.values()): logging.error('SCOPED_AUTH_KEYS values are not unique.') _settings_failures = True diff --git a/confidant/utils/misc.py b/confidant/utils/misc.py index c6673496..433711d5 100644 --- a/confidant/utils/misc.py +++ b/confidant/utils/misc.py @@ -10,7 +10,7 @@ def dict_deep_update(a, b): :param b: Right hand side with values to pull in :type b: dict """ - for key, val in b.iteritems(): + for key, val in b.items(): if isinstance(a.get(key), dict) and isinstance(val, dict): dict_deep_update(a[key], val) else: diff --git a/piptools_requirements.in b/piptools_requirements.in deleted file mode 100644 index 9ec4a098..00000000 --- a/piptools_requirements.in +++ /dev/null @@ -1,36 +0,0 @@ -# cryptography is a package which provides cryptographic recipes and -# primitives to Python developers. -# License: BSD -# Upstream url: https://github.com/pyca/cryptography -# Use: For encryption -cryptography==2.3.0 - -# C Foreign Function Interface for Python. -# License: MIT -# Upstream url: https://bitbucket.org/cffi/cffi -# Use: Required by cryptography package to be installed. This should be removed -# once crytography updates their depenencies to require cffi>=1.10.0. -cffi==1.10.0 - -# Provides enhanced HTTPS support for httplib and urllib2 using PyOpenSSL -# License: BSD -# Upstream url: https://github.com/cedadev/ndg_httpsclient/ -# Use: Securing requests for python < 2.7.9. -ndg-httpsclient==0.4.2 - -# ASN.1 types and codecs -# License: BSD -# Upstream url: http://sourceforge.net/projects/pyasn1/ -# Use: Securing requests for python < 2.7.9. -pyasn1==0.4.4 - -# Python wrapper module around the OpenSSL library -# License: APL2 -# Upstream url: https://github.com/pyca/pyopenssl -# Use: Securing requests for python < 2.7.9. -pyOpenSSL==16.2.0 - -# Python 2 and 3 compatibility utilities -# License: MIT -# Upstream url: http://pypi.python.org/pypi/six/ -six==1.10.0 diff --git a/piptools_requirements.txt b/piptools_requirements.txt index 6f665af4..ea276353 100644 --- a/piptools_requirements.txt +++ b/piptools_requirements.txt @@ -5,17 +5,17 @@ # control run piptools.compile confidant # asn1crypto==0.24.0 # via cryptography -certifi==2018.4.16 -cffi==1.10.0 -cryptography==2.3 +certifi==2019.6.16 +cffi==1.12.3 # via cryptography +cryptography==2.7 enum34==1.1.6 # via cryptography -idna==2.6 +idna==2.8 ipaddress==1.0.22 # via cryptography -ndg-httpsclient==0.4.2 -pyasn1==0.4.4 -pycparser==2.18 # via cffi -pyopenssl==16.2.0 -six==1.10.0 +ndg-httpsclient==0.5.1 +pyasn1==0.4.6 +pycparser==2.19 # via cffi +pyopenssl==19.0.0 +six==1.12.0 # via cryptography, pyopenssl pip==9.0.3 setuptools==39.0.1 diff --git a/piptools_requirements3.txt b/piptools_requirements3.txt index a093282d..2f3803b1 100644 --- a/piptools_requirements3.txt +++ b/piptools_requirements3.txt @@ -4,15 +4,6 @@ # # control run piptools.compile confidant # -asn1crypto==0.24.0 # via cryptography -cffi==1.10.0 -cryptography==2.3 -idna==2.6 # via cryptography -ndg-httpsclient==0.4.2 -pyasn1==0.4.4 -pycparser==2.18 # via cffi -pyopenssl==16.2.0 -six==1.10.0 pip==9.0.3 setuptools==39.0.1 diff --git a/requirements.in b/requirements.in index 7ef54b5c..41339580 100644 --- a/requirements.in +++ b/requirements.in @@ -16,34 +16,34 @@ Flask-Script==2.0.5 # License: Apache2 # Upstream url: https://github.com/boto/botocore # Use: For boto3 -botocore==1.4.58 +botocore>=1.4.58 # Boto3 is the Amazon Web Services (AWS) Software Development Kit (SDK) # for Python. # License: Apache2 # Upstream url: https://github.com/boto/boto3 # Use: For KMS -boto3==1.4.0 +boto3>1.5.0 # A Pythonic interface for Amazon's DynamoDB that supports Python 2 and 3. # License: MIT # Upstream url: http://jlafon.io/pynamodb.html # Use: For DynamoDB storage -pynamodb==3.2.1 +pynamodb>=3.2.1,<4 # cryptography is a package which provides cryptographic recipes and # primitives to Python developers. # License: BSD # Upstream url: https://github.com/pyca/cryptography # Use: For encryption -cryptography==2.3.0 +cryptography>2.3.0 # C Foreign Function Interface for Python. # License: MIT # Upstream url: https://bitbucket.org/cffi/cffi # Use: Required by cryptography package to be installed. This should be removed # once crytography updates their depenencies to require cffi>=1.10.0. -cffi==1.10.0 +cffi>1.10.0 # License: BSD # Upstream url: https://github.com/fengsp/flask-session @@ -75,7 +75,7 @@ Authomatic==0.1.0.post1 # License: MIT # Upstream url: https://github.com/onelogin/python3-saml # Use: For user authentication via SAML -python3-saml==1.4.0 +python3-saml>1.4.0 # Python HTTP for Humans. # License: Apache2 @@ -87,19 +87,19 @@ requests==2.11.1 # License: BSD # Upstream url: https://github.com/cedadev/ndg_httpsclient/ # Use: Securing requests for python < 2.7.9. -ndg-httpsclient==0.4.2 +ndg-httpsclient>0.4.2 # ASN.1 types and codecs # License: BSD # Upstream url: http://sourceforge.net/projects/pyasn1/ # Use: Securing requests for python < 2.7.9. -pyasn1==0.4.4 +pyasn1>0.4.4 # Python wrapper module around the OpenSSL library # License: APL2 # Upstream url: https://github.com/pyca/pyopenssl # Use: Securing requests for python < 2.7.9. -pyOpenSSL==16.2.0 +pyOpenSSL>16.2.0 # WSGI middleware to add CSP headers # License: Apache2 diff --git a/requirements.txt b/requirements.txt index 22eaa76f..e0a01d59 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,13 +6,13 @@ # asn1crypto==0.24.0 authomatic==0.1.0.post1 -boto3==1.4.0 -botocore==1.4.58 -certifi==2018.4.16 -cffi==1.10.0 +boto3==1.9.219 +botocore==1.12.219 +certifi==2019.6.16 +cffi==1.12.3 configparser==3.7.4 # via entrypoints, flake8 coverage==4.5.4 -cryptography==2.3 +cryptography==2.7 defusedxml==0.5.0 # via python3-saml docutils==0.13.1 # via botocore entrypoints==0.3 # via flake8 @@ -29,7 +29,7 @@ gevent==1.2.1 greenlet==0.4.12 guard==1.0.1 gunicorn==19.7.1 -idna==2.6 +idna==2.8 ipaddress==1.0.22 isodate==0.5.4 # via python3-saml itsdangerous==0.24 # via flask @@ -40,26 +40,27 @@ lxml==4.2.1 # via xmlsec markupsafe==1.0 # via jinja2 mccabe==0.6.1 # via flake8 mock==2.0.0 -ndg-httpsclient==0.4.2 +ndg-httpsclient==0.5.1 nose-pathmunge==0.1.2 nose==1.3.3 pbr==2.0.0 # via mock pkgconfig==1.3.1 # via xmlsec -pyasn1==0.4.4 +pyasn1==0.4.6 pycodestyle==2.5.0 # via flake8 -pycparser==2.18 +pycparser==2.19 pyflakes==2.1.1 # via flake8 -pynamodb==3.2.1 -pyopenssl==16.2.0 +pynamodb==3.4.1 +pyopenssl==19.0.0 python-dateutil==2.6.0 # via botocore, pynamodb -python3-saml==1.4.0 +python3-saml==1.7.0 pyyaml==3.11 redis==2.10.3 requests==2.11.1 -s3transfer==0.1.10 # via boto3 -six==1.10.0 +s3transfer==0.2.1 # via boto3 +six==1.12.0 statsd==3.2.1 typing==3.7.4 # via flake8 +urllib3==1.25.3 # via botocore werkzeug==0.12.1 # via flask xmlsec==1.3.3 # via python3-saml diff --git a/requirements3.txt b/requirements3.txt index 4c29ea11..eafb6fce 100644 --- a/requirements3.txt +++ b/requirements3.txt @@ -4,13 +4,13 @@ # # control run piptools.compile confidant # -asn1crypto==0.24.0 +asn1crypto==0.24.0 # via cryptography authomatic==0.1.0.post1 -boto3==1.4.0 -botocore==1.4.58 -cffi==1.10.0 +boto3==1.9.219 +botocore==1.12.219 +cffi==1.12.3 coverage==4.5.4 -cryptography==2.3 +cryptography==2.7 defusedxml==0.5.0 # via python3-saml docutils==0.13.1 # via botocore entrypoints==0.3 # via flake8 @@ -23,7 +23,6 @@ gevent==1.2.1 greenlet==0.4.12 guard==1.0.1 gunicorn==19.7.1 -idna==2.6 isodate==0.5.4 # via python3-saml itsdangerous==0.24 # via flask jinja2==2.9.6 # via flask @@ -33,25 +32,26 @@ lxml==4.2.1 # via xmlsec markupsafe==1.0 # via jinja2 mccabe==0.6.1 # via flake8 mock==2.0.0 -ndg-httpsclient==0.4.2 +ndg-httpsclient==0.5.1 nose-pathmunge==0.1.2 nose==1.3.3 pbr==2.0.0 # via mock pkgconfig==1.3.1 # via xmlsec -pyasn1==0.4.4 +pyasn1==0.4.6 pycodestyle==2.5.0 # via flake8 -pycparser==2.18 +pycparser==2.18 # via cffi pyflakes==2.1.1 # via flake8 -pynamodb==3.2.1 -pyopenssl==16.2.0 +pynamodb==3.4.1 +pyopenssl==19.0.0 python-dateutil==2.6.0 # via botocore, pynamodb -python3-saml==1.4.0 +python3-saml==1.7.0 pyyaml==3.11 redis==2.10.3 requests==2.11.1 -s3transfer==0.1.10 # via boto3 -six==1.10.0 +s3transfer==0.2.1 # via boto3 +six==1.10.0 # via cryptography, mock, pynamodb, pyopenssl, python-dateutil statsd==3.2.1 +urllib3==1.25.3 # via botocore werkzeug==0.12.1 # via flask xmlsec==1.3.3 # via python3-saml diff --git a/tests/integration/confidant/authnz/authnz_test.py b/tests/integration/confidant/authnz/authnz_test.py index 7ce17dd4..b4e00c93 100644 --- a/tests/integration/confidant/authnz/authnz_test.py +++ b/tests/integration/confidant/authnz/authnz_test.py @@ -130,12 +130,9 @@ def has_xsrf_cookie(headers): def test_invalid_kms_auth_token(self): app.debug = True app.config['USE_AUTH'] = True - auth = base64.b64encode( - '{0}:{1}'.format('confidant-development', 'faketoken') - ) - ret = self.test_client.open( + auth = base64.b64encode(b'confidant-development:faketoken').decode() + ret = self.test_client.get( '/v1/services/confidant-development', - 'GET', headers={'Authorization': 'Basic {0}'.format(auth)}, follow_redirects=False ) @@ -149,12 +146,9 @@ def test_valid_kms_auth_token(self, keymanager_mock): "payload": {"fake": "payload"}, "key_alias": "authnz-testing" } - auth = base64.b64encode( - '{0}:{1}'.format('confidant-development', 'faketoken') - ) - ret = self.test_client.open( + auth = base64.b64encode(b'confidant-development:faketoken').decode() + ret = self.test_client.get( '/v1/services/confidant-development', - 'GET', headers={'Authorization': 'Basic {0}'.format(auth)}, follow_redirects=False ) @@ -169,11 +163,10 @@ def test_valid_kms_auth_token_invalid_user(self, keymanager_mock): "key_alias": "authnz-testing" } auth = base64.b64encode( - '{0}:{1}'.format('confidant-development-baduser', 'faketoken') - ) - ret = self.test_client.open( + b'confidant-development-baduser:faketoken', + ).decode() + ret = self.test_client.get( '/v1/services/confidant-development', - 'GET', headers={'Authorization': 'Basic {0}'.format(auth)}, follow_redirects=False ) @@ -198,12 +191,9 @@ def test_valid_kms_auth_token_invalid_account(self, keymanager_mock): "payload": {"fake": "payload"}, "key_alias": "authnz-testing" } - auth = base64.b64encode( - '{0}:{1}'.format('confidant-development', 'faketoken') - ) - ret = self.test_client.open( + auth = base64.b64encode(b'confidant-development:faketoken').decode() + ret = self.test_client.get( '/v1/services/confidant-development', - 'GET', headers={'Authorization': 'Basic {0}'.format(auth)}, follow_redirects=False ) @@ -228,12 +218,9 @@ def test_valid_kms_auth_token_no_account(self, keymanager_mock): "payload": {"fake": "payload"}, "key_alias": "authnz-testing" } - auth = base64.b64encode( - '{0}:{1}'.format('confidant-development', 'faketoken') - ) - ret = self.test_client.open( + auth = base64.b64encode(b'confidant-development:faketoken').decode() + ret = self.test_client.get( '/v1/services/confidant-development', - 'GET', headers={'Authorization': 'Basic {0}'.format(auth)}, follow_redirects=False ) @@ -259,12 +246,9 @@ def test_valid_kms_auth_token_good_account(self, keymanager_mock): "payload": {"fake": "payload"}, "key_alias": "sandbox-auth-key" } - auth = base64.b64encode( - '{0}:{1}'.format('confidant-development', 'faketoken') - ) - ret = self.test_client.open( + auth = base64.b64encode(b'confidant-development:faketoken').decode() + ret = self.test_client.get( '/v1/services/confidant-development', - 'GET', headers={'Authorization': 'Basic {0}'.format(auth)}, follow_redirects=False ) diff --git a/tests/unit/confidant/ciphermanager_test.py b/tests/unit/confidant/ciphermanager_test.py index cebe11e0..1ee563d2 100644 --- a/tests/unit/confidant/ciphermanager_test.py +++ b/tests/unit/confidant/ciphermanager_test.py @@ -16,7 +16,7 @@ def test_cipher_version_2(self): self.assertNotEquals(ciphertext, 'testdata') cipher = CipherManager(key, 2) - plaintext = cipher.decrypt(ciphertext) + plaintext = cipher.decrypt(ciphertext).decode('UTF-8') # Assert that decrypting using a new cipher object with the same key # and version give back the same plaintext. diff --git a/tests/unit/confidant/keymanager_test.py b/tests/unit/confidant/keymanager_test.py index 93d2ddeb..6374f8c9 100644 --- a/tests/unit/confidant/keymanager_test.py +++ b/tests/unit/confidant/keymanager_test.py @@ -134,7 +134,7 @@ def test_decrypt_datakey_with_encryption(self, dmd_mock, dd_mock): app.config['USE_ENCRYPTION'] = True context = {'from': 'confidant-development', 'to': 'confidant-development'} - keymanager.decrypt_datakey('encrypted', context) + keymanager.decrypt_datakey(b'encrypted', context) # Assert that decrypt_datakey was called and decrypt_mock_datakey was # not called. From 49f88057009cd1e525863835606257be0c9c056a Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Mon, 9 Sep 2019 13:14:39 -0700 Subject: [PATCH 03/17] Fix email retrieval from session --- confidant/authnz/userauth.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/confidant/authnz/userauth.py b/confidant/authnz/userauth.py index b34c1b00..5a136162 100644 --- a/confidant/authnz/userauth.py +++ b/confidant/authnz/userauth.py @@ -205,6 +205,10 @@ def allowed_email_suffix(self): def check_authorization(self): email = self.current_email() + # during roll forward 2->3 this will be bytes when retrieved from + # session + if isinstance(email, bytes): + email = email.decode('UTF-8') if not self.passes_email_suffix(email): msg = 'User {!r} does not have email suffix {!r}'.format( From f523511913d3753d6c6b68119e90042c7f42fef7 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Mon, 9 Sep 2019 14:47:01 -0700 Subject: [PATCH 04/17] 2->3 email roll forward part 2 --- confidant/authnz/userauth.py | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/confidant/authnz/userauth.py b/confidant/authnz/userauth.py index 5a136162..cfb1c1ab 100644 --- a/confidant/authnz/userauth.py +++ b/confidant/authnz/userauth.py @@ -121,7 +121,9 @@ def set_current_user(self, email, first_name=None, last_name=None): } def current_email(self): - return self.current_user()['email'].lower() + ret = self.current_user()['email'].lower() + # when migrating from 2 -> 3, the session email object may be bytes + return ret.decode('UTF-8') if isinstance(ret, bytes) else ret def current_first_name(self): return self.current_user()['first_name'] @@ -205,10 +207,6 @@ def allowed_email_suffix(self): def check_authorization(self): email = self.current_email() - # during roll forward 2->3 this will be bytes when retrieved from - # session - if isinstance(email, bytes): - email = email.decode('UTF-8') if not self.passes_email_suffix(email): msg = 'User {!r} does not have email suffix {!r}'.format( @@ -235,7 +233,7 @@ def passes_email_whitelist(self, email): return True -class NullUserAuthenticator(object): +class NullUserAuthenticator(AbstractUserAuthenticator): """ Fake user authenticator class that performs no authentication. """ @@ -255,15 +253,6 @@ def current_user(self): 'last_name': 'user', } - def current_email(self): - return self.current_user()['email'].lower() - - def current_first_name(self): - return self.current_user()['first_name'] - - def current_last_name(self): - return self.current_user()['last_name'] - def is_authenticated(self): """Null users are always authenticated""" return True @@ -327,15 +316,6 @@ def current_user(self): return info - def current_email(self): - return self.current_user()['email'].lower() - - def current_first_name(self): - return self.current_user()['first_name'] - - def current_last_name(self): - return self.current_user()['last_name'] - def is_authenticated(self): """Any user that is able to make requests is authenticated""" self.assert_headers() From c89b4efd62123c2d27fc6fdeaa168b9ae6ec4d6d Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Mon, 9 Sep 2019 16:48:29 -0700 Subject: [PATCH 05/17] Have generate_value return text --- confidant/routes/v1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confidant/routes/v1.py b/confidant/routes/v1.py index ca42cfe8..35067531 100644 --- a/confidant/routes/v1.py +++ b/confidant/routes/v1.py @@ -1212,7 +1212,7 @@ def update_blind_credential(id): @app.route('/v1/value_generator', methods=['GET']) def generate_value(): value = kms_client.generate_random(NumberOfBytes=128)['Plaintext'] - value = base64.urlsafe_b64encode(value) + value = base64.urlsafe_b64encode(value).decode('UTF-8') value = re.sub(r'[\W_]+', '', value) if len(value) > VALUE_LENGTH: value = value[:VALUE_LENGTH] From d653ba43ade3ebc2ceae314b969518142fd2c85e Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 10 Sep 2019 15:56:33 -0400 Subject: [PATCH 06/17] Switch to kmsauth library for kmsauth support (#191) We split kmsauth out to a library a while ago, but hadn't switched confidant over to using it. This change uses kmsauth, rather than having the code in two spots. --- confidant/authnz/__init__.py | 94 ++++---- confidant/keymanager.py | 139 +----------- confidant/scripts/bootstrap.py | 2 +- confidant/settings.py | 6 +- .../source/basics/configuration.html.markdown | 6 +- piptools_requirements.txt | 3 - piptools_requirements3.txt | 3 - requirements.in | 8 +- requirements.txt | 4 +- requirements3.txt | 4 +- tests/unit/confidant/authnz/authnz_test.py | 95 --------- tests/unit/confidant/keymanager_test.py | 200 ------------------ 12 files changed, 63 insertions(+), 501 deletions(-) diff --git a/confidant/authnz/__init__.py b/confidant/authnz/__init__.py index 2cfb8399..5b324556 100644 --- a/confidant/authnz/__init__.py +++ b/confidant/authnz/__init__.py @@ -1,26 +1,47 @@ import fnmatch import logging +import kmsauth from flask import abort, request, g, make_response from flask import url_for from functools import wraps -from confidant import keymanager from confidant.app import app from confidant.utils import stats -from confidant.authnz.errors import (UserUnknownError, TokenVersionError, - AuthenticationError, NotAuthorized) +from confidant.authnz.errors import ( + UserUnknownError, + AuthenticationError, + NotAuthorized, +) from confidant.authnz import userauth PRIVILEGES = { 'user': ['*'], 'service': ['get_service'] } +_VALIDATOR = None user_mod = userauth.init_user_auth_class() +def _get_validator(): + global _VALIDATOR + if _VALIDATOR is None: + _VALIDATOR = kmsauth.KMSTokenValidator( + app.config['AUTH_KEY'], + app.config['USER_AUTH_KEY'], + app.config['AUTH_CONTEXT'], + app.config['AWS_DEFAULT_REGION'], + auth_token_max_lifetime=app.config['AUTH_TOKEN_MAX_LIFETIME'], + minimum_token_version=app.config['KMS_MINIMUM_TOKEN_VERSION'], + maximum_token_version=app.config['KMS_MAXIMUM_TOKEN_VERSION'], + scoped_auth_keys=app.config['SCOPED_AUTH_KEYS'], + token_cache_size=app.config['KMS_AUTH_TOKEN_CACHE_SIZE'], + ) + return _VALIDATOR + + def get_logged_in_user(): ''' Retrieve logged-in user's email that is stored in cache @@ -84,23 +105,6 @@ def decorated(*args, **kwargs): return decorated -def _parse_username(username): - username_arr = username.split('/') - if len(username_arr) == 3: - # V2 token format: version/service/myservice or version/user/myuser - version = int(username_arr[0]) - user_type = username_arr[1] - username = username_arr[2] - elif len(username_arr) == 1: - # Old format, specific to services: myservice - version = 1 - username = username_arr[0] - user_type = 'service' - else: - raise TokenVersionError('Unsupported username format.') - return version, user_type, username - - def _get_kms_auth_data(): data = {} auth = request.authorization @@ -108,18 +112,14 @@ def _get_kms_auth_data(): if auth and auth.get('username'): if not auth.get('password'): raise AuthenticationError('No password provided via basic auth.') - (data['version'], - data['user_type'], - data['from']) = _parse_username(auth['username']) + data['username'] = auth['username'] data['token'] = auth['password'] elif 'X-Auth-Token' in headers and 'X-Auth-From' in headers: if not headers.get('X-Auth-Token'): raise AuthenticationError( 'No X-Auth-Token provided via auth headers.' ) - (data['version'], - data['user_type'], - data['from']) = _parse_username(headers['X-Auth-From']) + data['username'] = headers['X-Auth-From'] data['token'] = headers['X-Auth-Token'] return data @@ -154,52 +154,50 @@ def decorated(*args, **kwargs): # User suppplied basic auth info try: kms_auth_data = _get_kms_auth_data() - except TokenVersionError: - logging.warning('Invalid token version used.') - return abort(403) except AuthenticationError: logging.exception('Failed to authenticate request.') return abort(403) if kms_auth_data: + validator = _get_validator() + _from = validator.extract_username_field( + kms_auth_data['username'], + 'from', + ) + _user_type = validator.extract_username_field( + kms_auth_data['username'], + 'user_type', + ) try: - if (kms_auth_data['user_type'] - not in app.config['KMS_AUTH_USER_TYPES']): + if _user_type not in app.config['KMS_AUTH_USER_TYPES']: msg = '{0} is not an allowed user type for KMS auth.' - msg = msg.format(kms_auth_data['user_type']) + msg = msg.format(_user_type) logging.warning(msg) return abort(403) with stats.timer('decrypt_token'): - token_data = keymanager.decrypt_token( - kms_auth_data['version'], - kms_auth_data['user_type'], - kms_auth_data['from'], + token_data = validator.decrypt_token( + kms_auth_data['username'], kms_auth_data['token'] ) logging.debug('Auth request had the following token_data:' ' {0}'.format(token_data)) msg = 'Authenticated {0} with user_type {1} via kms auth' - msg = msg.format( - kms_auth_data['from'], - kms_auth_data['user_type'] - ) + msg = msg.format(_from, _user_type) logging.debug(msg) - if user_type_has_privilege( - kms_auth_data['user_type'], - f.__name__): - g.user_type = kms_auth_data['user_type'] + if user_type_has_privilege(_user_type, f.__name__): + g.user_type = _user_type g.auth_type = 'kms' g.account = account_for_key_alias(token_data['key_alias']) - g.username = kms_auth_data['from'] + g.username = _from return f(*args, **kwargs) else: msg = '{0} is not authorized to access {1}.' - msg = msg.format(kms_auth_data['from'], f.__name__) + msg = msg.format(_from, f.__name__) logging.warning(msg) return abort(403) - except keymanager.TokenDecryptionError: + except kmsauth.TokenValidationError: logging.exception('Failed to decrypt authentication token.') msg = 'Access denied for {0}. Authentication Failed.' - msg = msg.format(kms_auth_data['from']) + msg = msg.format(_from) logging.warning(msg) return abort(403) diff --git a/confidant/keymanager.py b/confidant/keymanager.py index 8c0e29ab..b7c2dbb5 100644 --- a/confidant/keymanager.py +++ b/confidant/keymanager.py @@ -1,12 +1,8 @@ -import base64 import hashlib -import datetime -import json import logging import botocore from botocore.exceptions import ClientError -from lru import LRU import confidant.services from confidant.app import app @@ -29,40 +25,15 @@ iam_resource = confidant.services.get_boto_resource('iam') DATAKEYS = {} -SERVICEKEYS = {} -TOKENS = LRU(app.config['KMS_AUTH_TOKEN_CACHE_SIZE']) KEY_METADATA = {} -def get_key_arn(key_alias): - if key_alias not in KEY_METADATA: - KEY_METADATA[key_alias] = auth_kms_client.describe_key( - KeyId='alias/{0}'.format(key_alias) - ) - return KEY_METADATA[key_alias]['KeyMetadata']['Arn'] - - def get_key_id(key_alias): if key_alias not in KEY_METADATA: - KEY_METADATA[key_alias] = auth_kms_client.describe_key( - KeyId='alias/{0}'.format(key_alias) - ) + KEY_METADATA[key_alias] = auth_kms_client.describe_key(KeyId=key_alias) return KEY_METADATA[key_alias]['KeyMetadata']['KeyId'] -def get_key_alias_from_cache(key_arn): - ''' - Find a key's alias by looking up its key_arn in the KEY_METADATA - cache. This function will only work after a key has been lookedup by its - alias and is meant as a convenience function for turning an ARN that's - already been looked up back into its alias. - ''' - for alias in KEY_METADATA: - if KEY_METADATA[alias]['KeyMetadata']['Arn'] == key_arn: - return alias - return None - - def create_datakey(encryption_context): ''' Create a datakey from KMS. @@ -77,7 +48,7 @@ def create_datakey(encryption_context): stats.incr('at_rest_action', 2) return cryptolib.create_datakey( encryption_context, - 'alias/{0}'.format(app.config.get('KMS_MASTER_KEY')), + app.config.get('KMS_MASTER_KEY'), client=at_rest_kms_client ) @@ -105,108 +76,6 @@ def decrypt_datakey(data_key, encryption_context=None): return DATAKEYS[sha] -def valid_service_auth_key(key_arn): - if key_arn == get_key_arn(app.config['AUTH_KEY']): - return True - for key in app.config['SCOPED_AUTH_KEYS']: - if key_arn == get_key_arn(key): - return True - return False - - -def decrypt_token(version, user_type, _from, token): - ''' - Decrypt a token. - ''' - if (version > app.config['KMS_MAXIMUM_TOKEN_VERSION'] or - version < app.config['KMS_MINIMUM_TOKEN_VERSION']): - raise TokenDecryptionError('Unacceptable token version.') - stats.incr('token_version_{0}'.format(version)) - try: - token_key = '{0}{1}'.format( - hashlib.sha256(token.encode()).hexdigest(), - _from - ) - except Exception: - raise TokenDecryptionError('Authentication error.') - if token_key not in TOKENS: - try: - token = base64.b64decode(token) - context = { - # This key is sent to us. - 'to': app.config['AUTH_CONTEXT'], - # From a service. - 'from': _from - } - if version > 1: - context['user_type'] = user_type - with stats.timer('kms_decrypt_token'): - data = auth_kms_client.decrypt( - CiphertextBlob=token, - EncryptionContext=context - ) - # Decrypt doesn't take KeyId as an argument. We need to verify the - # correct key was used to do the decryption. - # Annoyingly, the KeyId from the data is actually an arn. - key_arn = data['KeyId'] - if user_type == 'service': - if not valid_service_auth_key(key_arn): - raise TokenDecryptionError( - 'Authentication error (wrong KMS key).' - ) - elif user_type == 'user': - if key_arn != get_key_arn(app.config['USER_AUTH_KEY']): - raise TokenDecryptionError( - 'Authentication error (wrong KMS key).' - ) - else: - raise TokenDecryptionError( - 'Authentication error. Unsupported user_type.' - ) - plaintext = data['Plaintext'] - payload = json.loads(plaintext) - key_alias = get_key_alias_from_cache(key_arn) - ret = {'payload': payload, 'key_alias': key_alias} - except TokenDecryptionError: - raise - # We don't care what exception is thrown. For paranoia's sake, fail - # here. - except Exception: - logging.exception('Failed to validate token.') - raise TokenDecryptionError('Authentication error. General error.') - else: - ret = TOKENS[token_key] - time_format = "%Y%m%dT%H%M%SZ" - now = datetime.datetime.utcnow() - try: - not_before = datetime.datetime.strptime( - ret['payload']['not_before'], - time_format - ) - not_after = datetime.datetime.strptime( - ret['payload']['not_after'], - time_format - ) - except Exception: - logging.exception( - 'Failed to get not_before and not_after from token payload.' - ) - raise TokenDecryptionError('Authentication error. Missing validity.') - delta = (not_after - not_before).seconds / 60 - if delta > app.config['AUTH_TOKEN_MAX_LIFETIME']: - logging.warning('Token used which exceeds max token lifetime.') - raise TokenDecryptionError( - 'Authentication error. Token lifetime exceeded.' - ) - if (now < not_before) or (now > not_after): - logging.warning('Invalid time validity for token.') - raise TokenDecryptionError( - 'Authentication error. Invalid time validity for token.' - ) - TOKENS[token_key] = ret - return TOKENS[token_key] - - def get_grants(): _grants = [] next_marker = None @@ -340,7 +209,3 @@ class ServiceGetGrantError(Exception): class ServiceCreateGrantError(Exception): pass - - -class TokenDecryptionError(Exception): - pass diff --git a/confidant/scripts/bootstrap.py b/confidant/scripts/bootstrap.py index 9c36003f..89e3cbe5 100644 --- a/confidant/scripts/bootstrap.py +++ b/confidant/scripts/bootstrap.py @@ -32,7 +32,7 @@ def run(self, _in, _out): secrets = f.read() data_key = cryptolib.create_datakey( {'type': 'bootstrap'}, - 'alias/{0}'.format(app.config['KMS_MASTER_KEY']) + app.config['KMS_MASTER_KEY'], ) f = Fernet(data_key['plaintext']) data = { diff --git a/confidant/settings.py b/confidant/settings.py index 17f47110..4d58ee05 100644 --- a/confidant/settings.py +++ b/confidant/settings.py @@ -226,7 +226,8 @@ def str_env(var_name, default=''): # as KMS_MASTER_KEY, but it's highly recommended to use a different key for # authentication and at-rest encryption. This key is specifically for the # 'service' role. -# Example: mykmskey +# If a key alias is used, rather than an ARN, it must be prefixed with: alias/ +# Example: alias/mykmskey AUTH_KEY = str_env('AUTH_KEY') # A dict of KMS key to account mappings. These keys are for the 'service' role # to support multiple AWS accounts. If services are scoped to accounts, @@ -238,6 +239,8 @@ def str_env(var_name, default=''): # The alias of the KMS key being used for authentication that is specifically # for the 'user' role. This should not be the same key as AUTH_KEY if your # kms token version is < 2, as it would allow services to masquerade as users. +# If a key alias is used, rather than an ARN, it must be prefixed with: alias/ +# Example: alias/mykmskey USER_AUTH_KEY = str_env('USER_AUTH_KEY') # The maximum lifetime of an authentication token in minutes. AUTH_TOKEN_MAX_LIFETIME = int_env('AUTH_TOKEN_MAX_LIFETIME', 60) @@ -332,6 +335,7 @@ def str_env(var_name, default=''): # Encryption # The KMS key to use for at-rest encryption for secrets in DynamoDB. +# If a key alias is used, rather than an ARN, it must be prefixed with: alias/ KMS_MASTER_KEY = str_env('KMS_MASTER_KEY') # Graphite events diff --git a/docs/source/basics/configuration.html.markdown b/docs/source/basics/configuration.html.markdown index 62d39c30..1b62ef5c 100644 --- a/docs/source/basics/configuration.html.markdown +++ b/docs/source/basics/configuration.html.markdown @@ -44,7 +44,7 @@ export AWS_DEFAULT_REGION='us-east-1' # The IAM role name of the confidant server. export AUTH_CONTEXT='confidant-production' # The KMS key used for auth. -export AUTH_KEY='authnz-production' +export AUTH_KEY='alias/authnz-production' # The DynamoDB table name for storage. export DYNAMODB_TABLE='confidant-production' # Auto-generate the dynamodb table. @@ -53,7 +53,7 @@ export DYNAMODB_CREATE_TABLE=true # https://github.com/surfly/gevent/issues/468 export GEVENT_RESOLVER='ares' # The KMS key used for at-rest encryption in DynamoDB. -export KMS_MASTER_KEY='confidant-production' +export KMS_MASTER_KEY='alias/confidant-production' # A long randomly generated string for CSRF protection. # SESSION_SECRET can be loaded via SECRETS_BOOTSTRAP export SESSION_SECRET='aBVmJA3zv6zWGjrYto135hkdox6mW2kOu7UaXIHK8ztJvT8w5O' @@ -444,7 +444,7 @@ will only allow service authentication. # for the 'user' role. This should not be the same key as AUTH_KEY if your # kms token version is less than 2, as it would allow services to masquerade # as users. -export USER_AUTH_KEY='user-auth-key' +export USER_AUTH_KEY='alias/user-auth-key' # The maximum version of the authentication token accepted. export KMS_MAXIMUM_TOKEN_VERSION='2' # The minimum version of the authentication token accepted. You should set this diff --git a/piptools_requirements.txt b/piptools_requirements.txt index ea276353..0109f34e 100644 --- a/piptools_requirements.txt +++ b/piptools_requirements.txt @@ -1,8 +1,5 @@ # # This file is autogenerated by pip-compile -# To update, run: -# -# control run piptools.compile confidant # asn1crypto==0.24.0 # via cryptography certifi==2019.6.16 diff --git a/piptools_requirements3.txt b/piptools_requirements3.txt index 2f3803b1..9ba5c099 100644 --- a/piptools_requirements3.txt +++ b/piptools_requirements3.txt @@ -1,8 +1,5 @@ # # This file is autogenerated by pip-compile -# To update, run: -# -# control run piptools.compile confidant # pip==9.0.3 diff --git a/requirements.in b/requirements.in index 41339580..3d6aa5cc 100644 --- a/requirements.in +++ b/requirements.in @@ -156,7 +156,7 @@ greenlet==0.4.12 # Use: For metrics collection statsd==3.2.1 -# License: MIT -# Upstream url: https://github.com/amitdev/lru-dict -# Use: in-memory LRU for caching auth tokens and at-rest keys -lru-dict==1.1.6 +# License: Apache2 +# Upstream url: https://github.com/lyft/python-kmsauth +# Use: kmsauth support library +kmsauth==0.4.0 diff --git a/requirements.txt b/requirements.txt index e0a01d59..9a4f4a52 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,5 @@ # # This file is autogenerated by pip-compile -# To update, run: -# -# control run piptools.compile confidant # asn1crypto==0.24.0 authomatic==0.1.0.post1 @@ -35,6 +32,7 @@ isodate==0.5.4 # via python3-saml itsdangerous==0.24 # via flask jinja2==2.9.6 # via flask jmespath==0.9.2 # via boto3, botocore +kmsauth==0.4.0 lru-dict==1.1.6 lxml==4.2.1 # via xmlsec markupsafe==1.0 # via jinja2 diff --git a/requirements3.txt b/requirements3.txt index eafb6fce..b0ae7503 100644 --- a/requirements3.txt +++ b/requirements3.txt @@ -1,8 +1,5 @@ # # This file is autogenerated by pip-compile -# To update, run: -# -# control run piptools.compile confidant # asn1crypto==0.24.0 # via cryptography authomatic==0.1.0.post1 @@ -27,6 +24,7 @@ isodate==0.5.4 # via python3-saml itsdangerous==0.24 # via flask jinja2==2.9.6 # via flask jmespath==0.9.2 # via boto3, botocore +kmsauth==0.4.0 lru-dict==1.1.6 lxml==4.2.1 # via xmlsec markupsafe==1.0 # via jinja2 diff --git a/tests/unit/confidant/authnz/authnz_test.py b/tests/unit/confidant/authnz/authnz_test.py index 830d1cee..71b16ef5 100644 --- a/tests/unit/confidant/authnz/authnz_test.py +++ b/tests/unit/confidant/authnz/authnz_test.py @@ -100,101 +100,6 @@ def test_user_is_service(self): g_mock.username = 'confidant-unitttest' self.assertFalse(authnz.user_is_service('notconfidant-unitttest')) - def test__parse_username(self): - self.assertEqual( - authnz._parse_username('confidant-unittest'), - (1, 'service', 'confidant-unittest') - ) - self.assertEqual( - authnz._parse_username('2/service/confidant-unittest'), - (2, 'service', 'confidant-unittest') - ) - with self.assertRaisesRegexp( - authnz.TokenVersionError, - 'Unsupported username format.'): - authnz._parse_username('3/service/confidant-unittest/extratoken') - - def test__get_kms_auth_data(self): - with app.test_request_context('/v1/user/email'): - self.assertEqual( - authnz._get_kms_auth_data(), - {} - ) - with patch('confidant.authnz.request') as request_mock: - request_mock.authorization = { - 'username': 'confidant-unittest', - 'password': 'encrypted' - } - request_mock.headers = {} - self.assertEqual( - authnz._get_kms_auth_data(), - {'version': 1, - 'user_type': 'service', - 'from': 'confidant-unittest', - 'token': 'encrypted'} - ) - with patch('confidant.authnz.request') as request_mock: - request_mock.authorization = { - 'username': '2/user/testuser', - 'password': 'encrypted' - } - request_mock.headers = {} - self.assertEqual( - authnz._get_kms_auth_data(), - {'version': 2, - 'user_type': 'user', - 'from': 'testuser', - 'token': 'encrypted'} - ) - with patch('confidant.authnz.request') as request_mock: - request_mock.authorization = {} - request_mock.headers = { - 'X-Auth-From': 'confidant-unittest', - 'X-Auth-Token': 'encrypted' - } - self.assertEqual( - authnz._get_kms_auth_data(), - {'version': 1, - 'user_type': 'service', - 'from': 'confidant-unittest', - 'token': 'encrypted'} - ) - with patch('confidant.authnz.request') as request_mock: - request_mock.authorization = {} - request_mock.headers = { - 'X-Auth-From': '2/user/testuser', - 'X-Auth-Token': 'encrypted' - } - self.assertEqual( - authnz._get_kms_auth_data(), - {'version': 2, - 'user_type': 'user', - 'from': 'testuser', - 'token': 'encrypted'} - ) - with patch('confidant.authnz.request') as request_mock: - request_mock.authorization = { - 'username': 'confidant-unittest', - 'password': '' - } - request_mock.headers = {} - with self.assertRaisesRegexp( - authnz.AuthenticationError, - 'No password provided via basic auth.' - ): - authnz._get_kms_auth_data(), - with patch('confidant.authnz.request') as request_mock: - request_mock.authorization = {} - request_mock.headers = { - 'X-Auth-From': '2/user/testuser', - 'X-Auth-Token': '' - } - with self.assertRaisesRegexp( - authnz.AuthenticationError, - 'No X-Auth-Token provided via auth headers.' - ): - authnz._get_kms_auth_data(), - def test_redirect_to_logout_if_no_auth(self): mock_fn = Mock() mock_fn.__name__ = 'mock_fn' diff --git a/tests/unit/confidant/keymanager_test.py b/tests/unit/confidant/keymanager_test.py index 6374f8c9..e3abbf94 100644 --- a/tests/unit/confidant/keymanager_test.py +++ b/tests/unit/confidant/keymanager_test.py @@ -4,7 +4,6 @@ from mock import patch from mock import MagicMock -from lru import LRU # Prevent call to KMS during tests from confidant import settings @@ -25,26 +24,6 @@ def tearDown(self): app.config['USE_ENCRYPTION'] = self.use_encryption app.config['SCOPED_AUTH_KEYS'] = self.scoped_auth_keys - @patch('confidant.keymanager.KEY_METADATA', {}) - @patch('confidant.keymanager.auth_kms_client.describe_key') - def test_get_key_arn(self, kms_mock): - kms_mock.return_value = {'KeyMetadata': {'Arn': 'mocked:arn'}} - self.assertEqual( - keymanager.get_key_arn('mockalias'), - 'mocked:arn' - ) - - @patch( - 'confidant.keymanager.KEY_METADATA', - {'mockalias': {'KeyMetadata': {'Arn': 'mocked:arn'}}} - ) - @patch('confidant.keymanager.auth_kms_client.describe_key') - def test_get_key_arn_cached(self, kms_mock): - self.assertEqual( - keymanager.get_key_arn('mockalias'), - 'mocked:arn' - ) - @patch('confidant.keymanager.KEY_METADATA', {}) @patch('confidant.keymanager.auth_kms_client.describe_key') def test_get_key_id(self, kms_mock): @@ -88,25 +67,6 @@ def test_decrypt_datakey_mocked(self): # Ensure we get the same value out that we sent in. self.assertEquals(ret, 'mocked_fernet_key') - @patch( - 'confidant.keymanager.get_key_arn' - ) - def test_valid_service_auth_key(self, gka_mock): - # Test AUTH_KEY arn checking - gka_mock.side_effect = ['test::arn'] - self.assertTrue(keymanager.valid_service_auth_key('test::arn')) - # Test SCOPED_AUTH_KEYS arn checking. There's two items in the side - # effect because get_key_arn will be called twice: once for AUTH_KEY - # check (which will fail) and another for the SCOPED_AUTH_KEYS check. - gka_mock.side_effect = ['auth::key', 'test::arn'] - app.config['SCOPED_AUTH_KEYS'] = {'test-key': 'test-account'} - self.assertTrue(keymanager.valid_service_auth_key('test::arn')) - # Test failure mode, where both AUTH_KEY and SCOPED_AUTH_KEYS checks - # fail. We have two items in side effects because get_key_arn will be - # called twice. - gka_mock.side_effect = ['auth::key', 'test::arn'] - self.assertFalse(keymanager.valid_service_auth_key('bad::arn')) - @patch( 'confidant.keymanager.cryptolib.create_datakey' ) @@ -140,163 +100,3 @@ def test_decrypt_datakey_with_encryption(self, dmd_mock, dd_mock): # not called. self.assertTrue(dd_mock.called) self.assertFalse(dmd_mock.called) - - @patch( - 'confidant.keymanager.get_key_arn', - MagicMock(return_value='mocked') - ) - @patch( - 'confidant.keymanager.get_key_alias_from_cache', - MagicMock(return_value='authnz-testing') - ) - @patch('confidant.keymanager.at_rest_kms_client.decrypt') - def test_decrypt_token(self, kms_mock): - time_format = "%Y%m%dT%H%M%SZ" - now = datetime.datetime.utcnow() - not_before = now.strftime(time_format) - _not_after = now + datetime.timedelta(minutes=60) - not_after = _not_after.strftime(time_format) - payload = json.dumps({ - 'not_before': not_before, - 'not_after': not_after - }) - kms_mock.return_value = { - 'Plaintext': payload, - 'KeyId': 'mocked' - } - self.assertEqual( - keymanager.decrypt_token( - 1, - 'service', - 'confidant-unittest', - 'ZW5jcnlwdGVk' - ), - { - 'payload': json.loads(payload), - 'key_alias': 'authnz-testing' - } - ) - keymanager.TOKENS = LRU(4096) - self.assertEqual( - keymanager.decrypt_token( - 2, - 'user', - 'testuser', - 'ZW5jcnlwdGVk' - ), - { - 'payload': json.loads(payload), - 'key_alias': 'authnz-testing' - } - ) - keymanager.TOKENS = LRU(4096) - with self.assertRaisesRegexp( - keymanager.TokenDecryptionError, - 'Unacceptable token version.'): - keymanager.decrypt_token( - 3, - 'user', - 'testuser', - 'ZW5jcnlwdGVk' - ) - with self.assertRaisesRegexp( - keymanager.TokenDecryptionError, - 'Authentication error. Unsupported user_type.'): - keymanager.decrypt_token( - 2, - 'unsupported', - 'testuser', - 'ZW5jcnlwdGVk' - ) - # Missing KeyId, will cause an exception to be thrown - kms_mock.return_value = { - 'Plaintext': payload - } - with self.assertRaisesRegexp( - keymanager.TokenDecryptionError, - 'Authentication error. General error.'): - keymanager.decrypt_token( - 2, - 'service', - 'confidant-unittest', - 'ZW5jcnlwdGVk' - ) - # Payload missing not_before/not_after - empty_payload = json.dumps({}) - kms_mock.return_value = { - 'Plaintext': empty_payload, - 'KeyId': 'mocked' - } - with self.assertRaisesRegexp( - keymanager.TokenDecryptionError, - 'Authentication error. Missing validity.'): - keymanager.decrypt_token( - 2, - 'service', - 'confidant-unittest', - 'ZW5jcnlwdGVk' - ) - # lifetime of 0 will make every token invalid. testing for proper delta - # checking. - lifetime = app.config['AUTH_TOKEN_MAX_LIFETIME'] - app.config['AUTH_TOKEN_MAX_LIFETIME'] = 0 - kms_mock.return_value = { - 'Plaintext': payload, - 'KeyId': 'mocked' - } - with self.assertRaisesRegexp( - keymanager.TokenDecryptionError, - 'Authentication error. Token lifetime exceeded.'): - keymanager.decrypt_token( - 2, - 'service', - 'confidant-unittest', - 'ZW5jcnlwdGVk' - ) - app.config['AUTH_TOKEN_MAX_LIFETIME'] = lifetime - # Token too old - now = datetime.datetime.utcnow() - _not_before = now - datetime.timedelta(minutes=60) - not_before = _not_before.strftime(time_format) - _not_after = now - datetime.timedelta(minutes=1) - not_after = _not_after.strftime(time_format) - payload = json.dumps({ - 'not_before': not_before, - 'not_after': not_after - }) - kms_mock.return_value = { - 'Plaintext': payload, - 'KeyId': 'mocked' - } - with self.assertRaisesRegexp( - keymanager.TokenDecryptionError, - 'Authentication error. Invalid time validity for token'): - keymanager.decrypt_token( - 2, - 'service', - 'confidant-unittest', - 'ZW5jcnlwdGVk' - ) - # Token too young - now = datetime.datetime.utcnow() - _not_before = now + datetime.timedelta(minutes=60) - not_before = _not_before.strftime(time_format) - _not_after = now + datetime.timedelta(minutes=120) - not_after = _not_after.strftime(time_format) - payload = json.dumps({ - 'not_before': not_before, - 'not_after': not_after - }) - kms_mock.return_value = { - 'Plaintext': payload, - 'KeyId': 'mocked' - } - with self.assertRaisesRegexp( - keymanager.TokenDecryptionError, - 'Authentication error. Invalid time validity for token'): - keymanager.decrypt_token( - 2, - 'service', - 'confidant-unittest', - 'ZW5jcnlwdGVk' - ) From 5f27104e7d68a8b67a0fc06ca45421188d0d6df1 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 10 Sep 2019 16:36:57 -0400 Subject: [PATCH 07/17] Remove kmsauth int tests (#195) --- setup.cfg | 2 +- .../confidant/authnz/authnz_test.py | 117 ------------------ 2 files changed, 1 insertion(+), 118 deletions(-) diff --git a/setup.cfg b/setup.cfg index aec40014..039a3830 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,7 +13,7 @@ xunit-file = build/nosetests.xml cover-package = confidant cover-xml = 1 cover-xml-file = build/coverage.xml -cover-min-percentage = 45 +cover-min-percentage = 43 [flake8] # The jenkins violations plugin can read the pylint format. diff --git a/tests/integration/confidant/authnz/authnz_test.py b/tests/integration/confidant/authnz/authnz_test.py index b4e00c93..e295100c 100644 --- a/tests/integration/confidant/authnz/authnz_test.py +++ b/tests/integration/confidant/authnz/authnz_test.py @@ -137,120 +137,3 @@ def test_invalid_kms_auth_token(self): follow_redirects=False ) self.assertEquals(ret.status_code, 403) - - @patch('confidant.keymanager.decrypt_token') - def test_valid_kms_auth_token(self, keymanager_mock): - app.debug = True - app.config['USE_AUTH'] = True - keymanager_mock.return_value = { - "payload": {"fake": "payload"}, - "key_alias": "authnz-testing" - } - auth = base64.b64encode(b'confidant-development:faketoken').decode() - ret = self.test_client.get( - '/v1/services/confidant-development', - headers={'Authorization': 'Basic {0}'.format(auth)}, - follow_redirects=False - ) - self.assertEquals(ret.status_code, 404) - - @patch('confidant.keymanager.decrypt_token') - def test_valid_kms_auth_token_invalid_user(self, keymanager_mock): - app.debug = True - app.config['USE_AUTH'] = True - keymanager_mock.return_value = { - "payload": {"fake": "payload"}, - "key_alias": "authnz-testing" - } - auth = base64.b64encode( - b'confidant-development-baduser:faketoken', - ).decode() - ret = self.test_client.get( - '/v1/services/confidant-development', - headers={'Authorization': 'Basic {0}'.format(auth)}, - follow_redirects=False - ) - self.assertEquals(ret.status_code, 401) - - @patch('confidant.keymanager.decrypt_token') - def test_valid_kms_auth_token_invalid_account(self, keymanager_mock): - app.debug = True - app.config['USE_AUTH'] = True - service = Service( - id='confidant-development', - data_type='service', - credentials=[], - blind_credentials=[], - account='sandbox', - enabled=True, - revision=1, - modified_by='test-user' - ) - service.save() - keymanager_mock.return_value = { - "payload": {"fake": "payload"}, - "key_alias": "authnz-testing" - } - auth = base64.b64encode(b'confidant-development:faketoken').decode() - ret = self.test_client.get( - '/v1/services/confidant-development', - headers={'Authorization': 'Basic {0}'.format(auth)}, - follow_redirects=False - ) - service.delete() - self.assertEquals(ret.status_code, 401) - - @patch('confidant.keymanager.decrypt_token') - def test_valid_kms_auth_token_no_account(self, keymanager_mock): - app.debug = True - app.config['USE_AUTH'] = True - service = Service( - id='confidant-development', - data_type='service', - credentials=[], - blind_credentials=[], - enabled=True, - revision=1, - modified_by='test-user' - ) - service.save() - keymanager_mock.return_value = { - "payload": {"fake": "payload"}, - "key_alias": "authnz-testing" - } - auth = base64.b64encode(b'confidant-development:faketoken').decode() - ret = self.test_client.get( - '/v1/services/confidant-development', - headers={'Authorization': 'Basic {0}'.format(auth)}, - follow_redirects=False - ) - service.delete() - self.assertEquals(ret.status_code, 200) - - @patch('confidant.keymanager.decrypt_token') - def test_valid_kms_auth_token_good_account(self, keymanager_mock): - app.debug = True - app.config['USE_AUTH'] = True - service = Service( - id='confidant-development', - data_type='service', - account='sandbox', - credentials=[], - blind_credentials=[], - enabled=True, - revision=1, - modified_by='test-user' - ) - service.save() - keymanager_mock.return_value = { - "payload": {"fake": "payload"}, - "key_alias": "sandbox-auth-key" - } - auth = base64.b64encode(b'confidant-development:faketoken').decode() - ret = self.test_client.get( - '/v1/services/confidant-development', - headers={'Authorization': 'Basic {0}'.format(auth)}, - follow_redirects=False - ) - service.delete() - self.assertEquals(ret.status_code, 200) From 729fd474b366e6ce127bb586c21f9b91b6a5ca0a Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 11 Sep 2019 12:56:11 -0400 Subject: [PATCH 08/17] Bump kmsauth and pass in stats (#196) --- confidant/authnz/__init__.py | 1 + requirements.in | 2 +- requirements.txt | 2 +- requirements3.txt | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/confidant/authnz/__init__.py b/confidant/authnz/__init__.py index 5b324556..afee3627 100644 --- a/confidant/authnz/__init__.py +++ b/confidant/authnz/__init__.py @@ -38,6 +38,7 @@ def _get_validator(): maximum_token_version=app.config['KMS_MAXIMUM_TOKEN_VERSION'], scoped_auth_keys=app.config['SCOPED_AUTH_KEYS'], token_cache_size=app.config['KMS_AUTH_TOKEN_CACHE_SIZE'], + stats=stats, ) return _VALIDATOR diff --git a/requirements.in b/requirements.in index 3d6aa5cc..04b58db7 100644 --- a/requirements.in +++ b/requirements.in @@ -159,4 +159,4 @@ statsd==3.2.1 # License: Apache2 # Upstream url: https://github.com/lyft/python-kmsauth # Use: kmsauth support library -kmsauth==0.4.0 +kmsauth==0.5.0 diff --git a/requirements.txt b/requirements.txt index 9a4f4a52..f84cc84e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ isodate==0.5.4 # via python3-saml itsdangerous==0.24 # via flask jinja2==2.9.6 # via flask jmespath==0.9.2 # via boto3, botocore -kmsauth==0.4.0 +kmsauth==0.5.0 lru-dict==1.1.6 lxml==4.2.1 # via xmlsec markupsafe==1.0 # via jinja2 diff --git a/requirements3.txt b/requirements3.txt index b0ae7503..f61bfc2d 100644 --- a/requirements3.txt +++ b/requirements3.txt @@ -24,7 +24,7 @@ isodate==0.5.4 # via python3-saml itsdangerous==0.24 # via flask jinja2==2.9.6 # via flask jmespath==0.9.2 # via boto3, botocore -kmsauth==0.4.0 +kmsauth==0.5.0 lru-dict==1.1.6 lxml==4.2.1 # via xmlsec markupsafe==1.0 # via jinja2 From 018ebfcc3a46716fd3baee3b5c8d692bf36f4189 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 11 Sep 2019 16:16:20 -0400 Subject: [PATCH 09/17] Update requirements (#197) --- requirements.in | 8 ++++---- requirements.txt | 49 ++++++++++++++++++++++++----------------------- requirements3.txt | 49 +++++++++++++++++++++++++---------------------- 3 files changed, 55 insertions(+), 51 deletions(-) diff --git a/requirements.in b/requirements.in index 04b58db7..f120e646 100644 --- a/requirements.in +++ b/requirements.in @@ -3,7 +3,7 @@ # License: BSD # Upstream url: http://github.com/mitsuhiko/flask/ # Use: For API. -Flask==0.10.1 +Flask==1.1.1 # Flask Scripting support for Flask # License: BSD @@ -81,7 +81,7 @@ python3-saml>1.4.0 # License: Apache2 # Upstream url: http://python-requests.org # Use: REST calls to external services -requests==2.11.1 +requests>=2.22.0,<3.0.0 # Provides enhanced HTTPS support for httplib and urllib2 using PyOpenSSL # License: BSD @@ -134,12 +134,12 @@ mock==2.0.0 # License: MIT # Upstream url: http://pyyaml.org/wiki/PyYAML # Use: For parsing users.yaml -PyYAML==3.11 +PyYAML==5.1.2 # License: MIT # Upstream url: http://gunicorn.org/ # Use: For wsgi running -gunicorn>=19.1.1,<20.0.0 +gunicorn>=19.9.0,<20.0.0 # License: MIT # Upstream url: http://www.gevent.org/ diff --git a/requirements.txt b/requirements.txt index f84cc84e..d601714e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,63 +3,64 @@ # asn1crypto==0.24.0 authomatic==0.1.0.post1 -boto3==1.9.219 -botocore==1.12.219 +boto3==1.9.227 +botocore==1.12.227 certifi==2019.6.16 cffi==1.12.3 -configparser==3.7.4 # via entrypoints, flake8 +chardet==3.0.4 # via requests +click==7.0 # via flask +configparser==4.0.1 # via entrypoints, flake8 coverage==4.5.4 cryptography==2.7 -defusedxml==0.5.0 # via python3-saml -docutils==0.13.1 # via botocore +defusedxml==0.6.0 # via python3-saml +docutils==0.15.2 # via botocore entrypoints==0.3 # via flake8 enum34==1.1.6 flake8==3.7.8 flask-script==2.0.5 flask-session==0.2.1 flask-sslify==0.1.5 -flask==0.10.1 +flask==1.1.1 funcsigs==1.0.2 # via mock functools32==3.2.3.post2 # via flake8 -futures==3.0.5 # via s3transfer +futures==3.3.0 # via s3transfer gevent==1.2.1 greenlet==0.4.12 guard==1.0.1 -gunicorn==19.7.1 +gunicorn==19.9.0 idna==2.8 ipaddress==1.0.22 -isodate==0.5.4 # via python3-saml -itsdangerous==0.24 # via flask -jinja2==2.9.6 # via flask -jmespath==0.9.2 # via boto3, botocore +isodate==0.6.0 # via python3-saml +itsdangerous==1.1.0 # via flask +jinja2==2.10.1 # via flask +jmespath==0.9.4 # via boto3, botocore kmsauth==0.5.0 -lru-dict==1.1.6 -lxml==4.2.1 # via xmlsec -markupsafe==1.0 # via jinja2 +lxml==4.4.1 # via xmlsec +markupsafe==1.1.1 # via jinja2 mccabe==0.6.1 # via flake8 mock==2.0.0 ndg-httpsclient==0.5.1 nose-pathmunge==0.1.2 nose==1.3.3 -pbr==2.0.0 # via mock -pkgconfig==1.3.1 # via xmlsec +pbr==5.4.3 # via mock +pkgconfig==1.5.1 # via xmlsec pyasn1==0.4.6 pycodestyle==2.5.0 # via flake8 pycparser==2.19 pyflakes==2.1.1 # via flake8 pynamodb==3.4.1 pyopenssl==19.0.0 -python-dateutil==2.6.0 # via botocore, pynamodb -python3-saml==1.7.0 -pyyaml==3.11 +python-dateutil==2.8.0 # via botocore, pynamodb +python3-saml==1.8.0 +pyyaml==5.1.2 redis==2.10.3 -requests==2.11.1 +requests==2.22.0 s3transfer==0.2.1 # via boto3 six==1.12.0 statsd==3.2.1 -typing==3.7.4 # via flake8 -urllib3==1.25.3 # via botocore -werkzeug==0.12.1 # via flask +typing==3.7.4.1 # via flake8 +urllib3==1.25.3 # via botocore, requests +werkzeug==0.15.6 # via flask xmlsec==1.3.3 # via python3-saml pip==9.0.3 diff --git a/requirements3.txt b/requirements3.txt index f61bfc2d..6eaa4726 100644 --- a/requirements3.txt +++ b/requirements3.txt @@ -3,54 +3,57 @@ # asn1crypto==0.24.0 # via cryptography authomatic==0.1.0.post1 -boto3==1.9.219 -botocore==1.12.219 +boto3==1.9.227 +botocore==1.12.227 +certifi==2019.6.16 # via requests cffi==1.12.3 +chardet==3.0.4 # via requests +click==7.0 # via flask coverage==4.5.4 cryptography==2.7 -defusedxml==0.5.0 # via python3-saml -docutils==0.13.1 # via botocore +defusedxml==0.6.0 # via python3-saml +docutils==0.15.2 # via botocore entrypoints==0.3 # via flake8 flake8==3.7.8 flask-script==2.0.5 flask-session==0.2.1 flask-sslify==0.1.5 -flask==0.10.1 +flask==1.1.1 gevent==1.2.1 greenlet==0.4.12 guard==1.0.1 -gunicorn==19.7.1 -isodate==0.5.4 # via python3-saml -itsdangerous==0.24 # via flask -jinja2==2.9.6 # via flask -jmespath==0.9.2 # via boto3, botocore +gunicorn==19.9.0 +idna==2.8 # via requests +isodate==0.6.0 # via python3-saml +itsdangerous==1.1.0 # via flask +jinja2==2.10.1 # via flask +jmespath==0.9.4 # via boto3, botocore kmsauth==0.5.0 -lru-dict==1.1.6 -lxml==4.2.1 # via xmlsec -markupsafe==1.0 # via jinja2 +lxml==4.4.1 # via xmlsec +markupsafe==1.1.1 # via jinja2 mccabe==0.6.1 # via flake8 mock==2.0.0 ndg-httpsclient==0.5.1 nose-pathmunge==0.1.2 nose==1.3.3 -pbr==2.0.0 # via mock -pkgconfig==1.3.1 # via xmlsec +pbr==5.4.3 # via mock +pkgconfig==1.5.1 # via xmlsec pyasn1==0.4.6 pycodestyle==2.5.0 # via flake8 -pycparser==2.18 # via cffi +pycparser==2.19 # via cffi pyflakes==2.1.1 # via flake8 pynamodb==3.4.1 pyopenssl==19.0.0 -python-dateutil==2.6.0 # via botocore, pynamodb -python3-saml==1.7.0 -pyyaml==3.11 +python-dateutil==2.8.0 # via botocore, pynamodb +python3-saml==1.8.0 +pyyaml==5.1.2 redis==2.10.3 -requests==2.11.1 +requests==2.22.0 s3transfer==0.2.1 # via boto3 -six==1.10.0 # via cryptography, mock, pynamodb, pyopenssl, python-dateutil +six==1.12.0 # via cryptography, isodate, mock, pynamodb, pyopenssl, python-dateutil statsd==3.2.1 -urllib3==1.25.3 # via botocore -werkzeug==0.12.1 # via flask +urllib3==1.25.3 # via botocore, requests +werkzeug==0.15.6 # via flask xmlsec==1.3.3 # via python3-saml pip==9.0.3 From bb3434aebcaa189075524d890b0c9722a7e33994 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 11 Sep 2019 17:03:05 -0400 Subject: [PATCH 10/17] Fix references to flask script for updates reqs (#198) --- confidant/scripts/bootstrap.py | 3 +-- confidant/scripts/manage.py | 2 +- confidant/scripts/migrate.py | 2 +- confidant/scripts/migrate_bool.py | 2 +- confidant/scripts/utils.py | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/confidant/scripts/bootstrap.py b/confidant/scripts/bootstrap.py index 89e3cbe5..68cbcfc3 100644 --- a/confidant/scripts/bootstrap.py +++ b/confidant/scripts/bootstrap.py @@ -6,8 +6,7 @@ import yaml from cryptography.fernet import Fernet -from flask.ext.script import Command -from flask.ext.script import Option +from flask_script import Command, Option from confidant import settings from confidant.app import app diff --git a/confidant/scripts/manage.py b/confidant/scripts/manage.py index 53ac69e6..fe8718db 100644 --- a/confidant/scripts/manage.py +++ b/confidant/scripts/manage.py @@ -1,4 +1,4 @@ -from flask.ext.script import Manager +from flask_script import Manager from confidant import app from confidant.scripts.utils import ManageGrants diff --git a/confidant/scripts/migrate.py b/confidant/scripts/migrate.py index fc18aedb..b0c6f4ff 100644 --- a/confidant/scripts/migrate.py +++ b/confidant/scripts/migrate.py @@ -1,6 +1,6 @@ import sys import logging -from flask.ext.script import Command +from flask_script import Command from confidant.app import app from confidant.models.blind_credential import BlindCredential diff --git a/confidant/scripts/migrate_bool.py b/confidant/scripts/migrate_bool.py index 7d84e49f..cefaf2ba 100644 --- a/confidant/scripts/migrate_bool.py +++ b/confidant/scripts/migrate_bool.py @@ -1,6 +1,6 @@ from confidant.app import app -from flask.ext.script import Command, Option +from flask_script import Command, Option import time from botocore.exceptions import ClientError diff --git a/confidant/scripts/utils.py b/confidant/scripts/utils.py index 53e119d2..7ee92c2e 100644 --- a/confidant/scripts/utils.py +++ b/confidant/scripts/utils.py @@ -1,6 +1,6 @@ import sys import logging -from flask.ext.script import Command +from flask_script import Command from botocore.exceptions import ClientError import confidant.services From 3471da35aa15ae11380af1b619d489da61638813 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Thu, 12 Sep 2019 10:25:27 -0400 Subject: [PATCH 11/17] Update configparser, since the frozen version was unpublished --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d601714e..edfdd57b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ certifi==2019.6.16 cffi==1.12.3 chardet==3.0.4 # via requests click==7.0 # via flask -configparser==4.0.1 # via entrypoints, flake8 +configparser==4.0.2 # via entrypoints, flake8 coverage==4.5.4 cryptography==2.7 defusedxml==0.6.0 # via python3-saml From 7f4c19beae0e52ed4e8ee6a5742e63e1ed666c5e Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Thu, 12 Sep 2019 14:48:37 -0400 Subject: [PATCH 12/17] Bump version to 5.0.0 --- CHANGELOG.md | 22 ++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b14b04de..cae93542 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## 5.0.0 + +* This is a breaking release. This release slightly changes the values needed + for the ``AUTH_KEY``, ``USER_AUTH_KEY``, and ``KMS_MASTER_KEY`` settings. + The previous way these settings were set were to use the alias name, without + an ``alias/`` prefix. In this release we switched to using the kmsauth + library for kms authentication support, which supports aliases and ARNs for + keys, which means that for these three settings, it's necessary to add an + ``alias/`` prefix to the value. So, for example, if your setting was + ``my-auth-key``, the new value would be ``alias/my-auth-key``. Though this + change of behavior was limited to kmsauth, for consistency we also changed + ``KMS_MASTER_KEY`` to use the same behavior. For all three settings, it's + also now possible to use ARNs as values, instead of just key aliases. +* confidant now supports python2 and python3. +* Requirements have been updated to resolve some reported security + vulnerabilities in a few of the frozen requirements. A library affecting + user sessions was upgraded which will cause users to be logged out after + upgrade, which means if you're doing a rolling upgrade, that during the + upgrade, you may have users that seemingly randomly get logged out. After + a finished upgrade, users should only be logged out once, if they're + currently logged in. + ## 4.4.0 * Use ``dict`` and ``set`` in pynamo models rather than ``{}`` and ``set()``, diff --git a/setup.py b/setup.py index 53354a33..f8b8e937 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ setup( name="confidant", - version="4.4.0", + version="5.0.0", packages=find_packages(exclude=["test*"]), include_package_data=True, zip_safe=False, From 3f04568883b50aa7fcfeb41683db4b36701b7038 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Thu, 12 Sep 2019 15:35:41 -0400 Subject: [PATCH 13/17] Bump to 5.0.1 --- CHANGELOG.md | 4 ++++ docker_push.sh | 4 ++-- setup.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cae93542..d0a97aca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 5.0.1 + +* Packaging fixes for docker + ## 5.0.0 * This is a breaking release. This release slightly changes the values needed diff --git a/docker_push.sh b/docker_push.sh index 7c434a90..3875cbc3 100755 --- a/docker_push.sh +++ b/docker_push.sh @@ -1,14 +1,14 @@ #!/bin/bash if [ "$TRAVIS_PULL_REQUEST" == "false" -a -n "$TRAVIS_TAG" ] then - docker login -e $DOCKER_EMAIL -u $DOCKER_USERNAME -p $DOCKER_PASSWORD + docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD export TAG="$TRAVIS_TAG" echo "TAG is $TAG" docker tag $TRAVIS_REPO_SLUG:$TRAVIS_COMMIT $REPO:$TAG docker push $TRAVIS_REPO_SLUG:$TAG elif [ "$TRAVIS_PULL_REQUEST" == "false" -a "$TRAVIS_BRANCH" == "master" ] then - docker login -e $DOCKER_EMAIL -u $DOCKER_USERNAME -p $DOCKER_PASSWORD + docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD export TAG="latest" echo "TAG is $TAG" docker tag $TRAVIS_REPO_SLUG:$TRAVIS_COMMIT $REPO:$TAG diff --git a/setup.py b/setup.py index f8b8e937..f6b36535 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ setup( name="confidant", - version="5.0.0", + version="5.0.1", packages=find_packages(exclude=["test*"]), include_package_data=True, zip_safe=False, From 108a39bd2254689523217d5b80b889d6a4009da9 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Mon, 23 Sep 2019 11:35:18 -0700 Subject: [PATCH 14/17] Fix USE_ENCRYPTION=false in python3 --- confidant/ciphermanager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/confidant/ciphermanager.py b/confidant/ciphermanager.py index 28a48656..62017a98 100644 --- a/confidant/ciphermanager.py +++ b/confidant/ciphermanager.py @@ -25,7 +25,9 @@ def encrypt(self, raw): logging.warning('Not using encryption in CipherManager.encrypt' ' If you are not running in a development or test' ' environment, this should not be happening!') - return 'DANGER_NOT_ENCRYPTED_{0}'.format(base64.b64encode(raw)) + return 'DANGER_NOT_ENCRYPTED_{0}'.format( + base64.b64encode(raw.encode('UTF-8')).decode('UTF-8'), + ) if self.version == 2: f = Fernet(self.key) return f.encrypt(raw.encode('utf-8')).decode('UTF-8') From 57532d68849a0f44542945af1d5bf83cb57f7612 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Mon, 23 Sep 2019 12:29:48 -0700 Subject: [PATCH 15/17] Bump version to 5.1.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f6b36535..9a120fb9 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ setup( name="confidant", - version="5.0.1", + version="5.1.0", packages=find_packages(exclude=["test*"]), include_package_data=True, zip_safe=False, From 6a0d3a6c22affe50f5d2e5b3cdcf4a099bf11424 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 16 Oct 2019 11:02:01 -0700 Subject: [PATCH 16/17] Open pem file as binary (#203) * Open pem file as binary * Change travis image to bionic * Fix invalid option * Remove --no-ri --- .travis.yml | 4 ++-- confidant/lib/cryptolib.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index cc0059b0..937cf73f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: xenial +dist: bionic language: python env: - REPO=lyft/confidant @@ -9,7 +9,7 @@ before_install: - docker build -f Dockerfile -t $REPO:$TRAVIS_COMMIT . - docker run -v $PWD/confidant/dist:/tmp/dist $REPO:$TRAVIS_COMMIT /bin/sh -c "cp -r /srv/confidant/confidant/dist/. /tmp/dist/." install: - - gem install travis --no-rdoc --no-ri + - gem install travis --no-document script: - travis lint .travis.yml --skip-completion-check - docker run $REPO:$TRAVIS_COMMIT /bin/sh -c "make test" diff --git a/confidant/lib/cryptolib.py b/confidant/lib/cryptolib.py index 1871684d..6a0c1d97 100644 --- a/confidant/lib/cryptolib.py +++ b/confidant/lib/cryptolib.py @@ -123,7 +123,7 @@ def load_private_key_pem(path, password=None): :returns: An RSA private key object. :rtype: cryptography.hazmat.primitives.asymmetric.rsa.RSAPrivateKey """ - with open(path, 'r') as f: + with open(path, 'rb') as f: return serialization.load_pem_private_key(f.read(), password=password, backend=default_backend()) From c8cbc2ba64f8093e6b47c2f2eff59f5d291d54fd Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 16 Oct 2019 11:06:23 -0700 Subject: [PATCH 17/17] Bump version to 5.2.0 --- CHANGELOG.md | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0a97aca..9a1b7aa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 5.2.0 + +* Python3 fix in function ``load_private_key_pem`` in ``confidant.lib.cryptolib`` + +## 5.1.0 + +* Python3 fix in class ``CipherManager`` in ``confidant.ciphermanager`` + ## 5.0.1 * Packaging fixes for docker diff --git a/setup.py b/setup.py index 9a120fb9..2cdfc979 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ setup( name="confidant", - version="5.1.0", + version="5.2.0", packages=find_packages(exclude=["test*"]), include_package_data=True, zip_safe=False,