fix kinesis error injection; fix S3 content md5 checks; fix S3 host h…

…eader (localstack#772)

localstack/services/generic_proxy.py

@@ -1,18 +1,18 @@
-from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
-import requests
 import os
 import sys
-import traceback
-import logging
 import ssl
-import inspect
 import socket
+import inspect
+import logging
+import traceback
+import requests
 from flask_cors import CORS
 from requests.structures import CaseInsensitiveDict
 from requests.models import Response, Request
 from six import iteritems
 from six.moves.socketserver import ThreadingMixIn
 from six.moves.urllib.parse import urlparse
+from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 from localstack.config import TMP_FOLDER, USE_SSL
 from localstack.constants import ENV_INTERNAL_TEST_RUN
 from localstack.utils.common import FuncThread, generate_ssl_cert, to_bytes

localstack/services/kinesis/kinesis_listener.py

@@ -21,7 +21,8 @@ def forward_request(self, method, path, data, headers):
         data = json.loads(to_str(data))
 
         if random.random() < config.KINESIS_ERROR_PROBABILITY:
-            return kinesis_error_response(data)
+            if headers.get('X-Amz-Target') in [ACTION_PUT_RECORD, ACTION_PUT_RECORDS]:
+                return kinesis_error_response(data)
         return True
 
     def return_response(self, method, path, data, headers, response):

localstack/services/s3/s3_listener.py

@@ -2,6 +2,8 @@
 import logging
 import json
 import uuid
+import base64
+import codecs
 import xmltodict
 import collections
 import six
@@ -279,6 +281,26 @@ def strip_chunk_signatures(data):
     return data_new
 
 
+def check_content_md5(data, headers):
+    actual = md5(data)
+    expected = headers['Content-MD5']
+    try:
+        expected = to_str(codecs.encode(base64.b64decode(expected), 'hex'))
+    except Exception:
+        expected = '__invalid__'
+    if actual != expected:
+        response = Response()
+        result = {
+            'Error': {
+                'Code': 'InvalidDigest',
+                'Message': 'The Content-MD5 you specified was invalid'
+            }
+        }
+        response._content = xmltodict.unparse(result)
+        response.status_code = 400
+        return response
+
+
 def expand_redirect_url(starting_url, key, bucket):
     """ Add key and bucket parameters to starting URL query string. """
     parsed = urlparse.urlparse(starting_url)
@@ -325,7 +347,6 @@ def get_bucket_name(path, headers):
         match = pattern.match(host)
         if match:
             bucket_name = match.groups()[0]
-
             break
 
     # we're either returning the original bucket_name,
@@ -337,6 +358,17 @@ class ProxyListenerS3(ProxyListener):
 
     def forward_request(self, method, path, data, headers):
 
+        # Make sure we use 'localhost' as forward host, to ensure moto uses path style addressing.
+        # Note that all S3 clients using LocalStack need to enable path style addressing.
+        if 's3.amazonaws.com' not in headers.get('host', ''):
+            headers['host'] = 'localhost'
+
+        # check content md5 hash integrity
+        if 'Content-MD5' in headers:
+            response = check_content_md5(data, headers)
+            if response is not None:
+                return response
+
         modified_data = None
 
         # If this request contains streaming v4 authentication signatures, strip them from the message
@@ -439,7 +471,7 @@ def return_response(self, method, path, data, headers, response):
 
         bucket_name = get_bucket_name(path, headers)
 
-        # No path-name based bucket name?  Try host-based
+        # No path-name based bucket name? Try host-based
         hostname_parts = headers['host'].split('.')
         if (not bucket_name or len(bucket_name) == 0) and len(hostname_parts) > 1:
             bucket_name = hostname_parts[0]

requirements.txt

@@ -17,7 +17,7 @@ flask_swagger==0.2.12
 jsonpath-rw==1.4.0
 localstack-ext>=0.8.6
 localstack-client==0.4
-moto-ext==1.2.0
+moto-ext==1.3.3
 nose==1.3.7
 psutil==5.4.3
 pyOpenSSL==17.0.0

tests/integration/test_s3.py

@@ -204,3 +204,20 @@ def test_s3_get_response_headers():
     # clean up
     s3_client.delete_objects(Bucket=bucket_name, Delete={'Objects': [{'Key': object_key}]})
     s3_client.delete_bucket(Bucket=bucket_name)
+
+
+def test_s3_invalid_content_md5():
+    bucket_name = 'test-bucket-%s' % short_uid()
+    s3_client = aws_stack.connect_to_service('s3')
+    s3_client.create_bucket(Bucket=bucket_name)
+
+    # put object with invalid content MD5
+    for hash in ['__invalid__', '000']:
+        raised = False
+        try:
+            s3_client.put_object(Bucket=bucket_name, Key='test-key',
+                Body='something', ContentMD5=hash)
+        except Exception:
+            raised = True
+        if not raised:
+            raise Exception('Invalid MD5 hash "%s" should have raised an error' % hash)