From 582f3cd5d6f58137fb3225ca26ddd8ddc28e97ee Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Tue, 22 May 2018 10:48:24 -0400 Subject: [PATCH] strip chunk signatures for S3 content md5 checks (#775) --- localstack/ext/java/pom.xml | 5 + .../localstack/docker/LocalstackDocker.java | 1 - localstack/services/install.py | 5 +- localstack/services/s3/s3_listener.py | 101 +++++++++--------- 4 files changed, 62 insertions(+), 50 deletions(-) diff --git a/localstack/ext/java/pom.xml b/localstack/ext/java/pom.xml index 61852bab27af3..e88911aa2e0ff 100644 --- a/localstack/ext/java/pom.xml +++ b/localstack/ext/java/pom.xml @@ -117,6 +117,11 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.7.0 + org.apache.maven.plugins maven-javadoc-plugin diff --git a/localstack/ext/java/src/main/java/cloud/localstack/docker/LocalstackDocker.java b/localstack/ext/java/src/main/java/cloud/localstack/docker/LocalstackDocker.java index b99c5207097ed..4c0722470596c 100644 --- a/localstack/ext/java/src/main/java/cloud/localstack/docker/LocalstackDocker.java +++ b/localstack/ext/java/src/main/java/cloud/localstack/docker/LocalstackDocker.java @@ -1,6 +1,5 @@ package cloud.localstack.docker; -import cloud.localstack.LocalstackTestRunner; import cloud.localstack.ServiceName; import cloud.localstack.docker.command.RegexStream; import cloud.localstack.docker.exception.LocalstackDockerException; diff --git a/localstack/services/install.py b/localstack/services/install.py index a9768b5a0cc6f..9e6652fc063c1 100755 --- a/localstack/services/install.py +++ b/localstack/services/install.py @@ -37,7 +37,10 @@ def install_elasticsearch(): mkdir(INSTALL_DIR_INFRA) # download and extract archive download_and_extract_with_retry(ELASTICSEARCH_JAR_URL, TMP_ARCHIVE_ES, INSTALL_DIR_INFRA) - run('cd %s && mv elasticsearch* elasticsearch' % (INSTALL_DIR_INFRA)) + elasticsearch_dir = glob.glob(os.path.join(INSTALL_DIR_INFRA, 'elasticsearch*')) + if not elasticsearch_dir: + raise Exception('Unable to find Elasticsearch folder in %s' % INSTALL_DIR_INFRA) + shutil.move(elasticsearch_dir[0], INSTALL_DIR_ES) for dir_name in ('data', 'logs', 'modules', 'plugins', 'config/scripts'): dir_path = '%s/%s' % (INSTALL_DIR_ES, dir_name) diff --git a/localstack/services/s3/s3_listener.py b/localstack/services/s3/s3_listener.py index 19be520eb7b5c..abe1536f6c67e 100644 --- a/localstack/services/s3/s3_listener.py +++ b/localstack/services/s3/s3_listener.py @@ -282,7 +282,7 @@ def strip_chunk_signatures(data): def check_content_md5(data, headers): - actual = md5(data) + actual = md5(strip_chunk_signatures(data)) expected = headers['Content-MD5'] try: expected = to_str(codecs.encode(base64.b64decode(expected), 'hex')) @@ -354,6 +354,56 @@ def get_bucket_name(path, headers): return bucket_name +def handle_notification_request(bucket, method, data): + response = Response() + response.status_code = 200 + response._content = '' + if method == 'GET': + # TODO check if bucket exists + result = '' % XMLNS_S3 + if bucket in S3_NOTIFICATIONS: + notif = S3_NOTIFICATIONS[bucket] + for dest in NOTIFICATION_DESTINATION_TYPES: + if dest in notif: + dest_dict = { + '%sConfiguration' % dest: { + 'Id': uuid.uuid4(), + dest: notif[dest], + 'Event': notif['Event'], + 'Filter': notif['Filter'] + } + } + result += xmltodict.unparse(dest_dict, full_document=False) + result += '' + response._content = result + + if method == 'PUT': + parsed = xmltodict.parse(data) + notif_config = parsed.get('NotificationConfiguration') + S3_NOTIFICATIONS.pop(bucket, None) + for dest in NOTIFICATION_DESTINATION_TYPES: + config = notif_config.get('%sConfiguration' % (dest)) + if config: + events = config.get('Event') + if isinstance(events, six.string_types): + events = [events] + event_filter = config.get('Filter', {}) + # make sure FilterRule is an array + s3_filter = _get_s3_filter(event_filter) + if s3_filter and not isinstance(s3_filter.get('FilterRule', []), list): + s3_filter['FilterRule'] = [s3_filter['FilterRule']] + # create final details dict + notification_details = { + 'Id': config.get('Id'), + 'Event': events, + dest: config.get(dest), + 'Filter': event_filter + } + # TODO: what if we have multiple destinations - would we overwrite the config? + S3_NOTIFICATIONS[bucket] = clone(notification_details) + return response + + class ProxyListenerS3(ProxyListener): def forward_request(self, method, path, data, headers): @@ -400,53 +450,8 @@ def forward_request(self, method, path, data, headers): bucket = path.split('/')[1] query_map = urlparse.parse_qs(query) if query == 'notification' or 'notification' in query_map: - response = Response() - response.status_code = 200 - if method == 'GET': - # TODO check if bucket exists - result = '' % XMLNS_S3 - if bucket in S3_NOTIFICATIONS: - notif = S3_NOTIFICATIONS[bucket] - for dest in NOTIFICATION_DESTINATION_TYPES: - if dest in notif: - dest_dict = { - '%sConfiguration' % dest: { - 'Id': uuid.uuid4(), - dest: notif[dest], - 'Event': notif['Event'], - 'Filter': notif['Filter'] - } - } - result += xmltodict.unparse(dest_dict, full_document=False) - result += '' - response._content = result - - if method == 'PUT': - parsed = xmltodict.parse(data) - notif_config = parsed.get('NotificationConfiguration') - S3_NOTIFICATIONS.pop(bucket, None) - for dest in NOTIFICATION_DESTINATION_TYPES: - config = notif_config.get('%sConfiguration' % (dest)) - if config: - events = config.get('Event') - if isinstance(events, six.string_types): - events = [events] - event_filter = config.get('Filter', {}) - # make sure FilterRule is an array - s3_filter = _get_s3_filter(event_filter) - if s3_filter and not isinstance(s3_filter.get('FilterRule', []), list): - s3_filter['FilterRule'] = [s3_filter['FilterRule']] - # create final details dict - notification_details = { - 'Id': config.get('Id'), - 'Event': events, - dest: config.get(dest), - 'Filter': event_filter - } - # TODO: what if we have multiple destinations - would we overwrite the config? - S3_NOTIFICATIONS[bucket] = clone(notification_details) - - # return response for ?notification request + # handle and return response for ?notification request + response = handle_notification_request(bucket, method, data) return response if query == 'cors' or 'cors' in query_map: