From 4d601a3c668c746141df4611e2c1e6f220553683 Mon Sep 17 00:00:00 2001 From: Bastien Date: Wed, 20 Sep 2023 21:14:25 +0100 Subject: [PATCH] Implementation of Ruff accross the project (#489) * resolving conflict * refactor orders service to python * add persistence to orders service with dynamodb * orders table cloudformation output * node-modules package-lock * add validation and more testing * ruff github workflow commit 1 * README build status * Ruff - autofix errors * ruff config * ruff: various small fixes * Ruff fix E722 no bare except * search for specific table during local and add more test * avoid E712 on generators * explicitly add routes and error handlers to carts service using blueprints rather than relying on import side effects * explicit routes and handlers * update valid keys * remove unused imports and scripts * update valid keys for order service * remove old go scripts and make order keys consistent * remove ttl order inherits from cart * Update app.py and remove unused imports * fix linter errors * fix linter errors-too long lines --------- Co-authored-by: Benedict Nartey-Tokoli Co-authored-by: Benedict Nartey-Tokoli <67244216+Adibuer-lab@users.noreply.github.com> --- .github/workflows/ruff.yml | 8 ++++ README.md | 6 +++ generators/datagenerator/amplitude.py | 2 - generators/datagenerator/file.py | 2 +- generators/datagenerator/funnel.py | 3 +- generators/datagenerator/output.py | 4 +- generators/datagenerator/segment.py | 2 +- generators/datagenerator/users.py | 8 +--- .../generate_interactions_personalize.py | 11 +++-- pyproject.toml | 48 +++++++++++++++++++ .../alexa-skill-lambda/alexa-skill-lambda.py | 13 ++--- .../ivs-create-channels.py | 4 +- .../location-geofence-event.py | 4 +- .../delete_dataset_groups.py | 2 +- .../pinpoint-auto-workshop.py | 16 ++++--- .../pinpoint-sms-alerts.py | 3 -- .../segment-personalize-events-destination.py | 10 ++-- ...gment-personalize-inference-destination.py | 8 ++-- src/carts/src/carts-service/dynamo_setup.py | 2 - src/carts/src/carts-service/services.py | 2 +- src/carts/test/integ/test_carts.py | 2 +- src/location/src/location-service/app.py | 2 +- src/offers/src/offers-service/app.py | 2 +- src/products/load_catalog.py | 3 +- .../src/recommendations-service/app.py | 22 ++++++--- .../experimentation/resolvers.py | 7 +-- .../experimentation/test_experiment.py | 1 - .../experimentation/test_resolvers.py | 1 - .../src/recommendations-service/run_tests.py | 4 +- .../test/integ/test_discounted.py | 1 - src/search/src/search-service/app.py | 5 +- src/users/test/integ/test-users.py | 1 - src/videos/src/videos-service/app.py | 19 ++++---- src/videos/src/videos-service/run_tests.py | 4 +- 34 files changed, 147 insertions(+), 85 deletions(-) create mode 100644 .github/workflows/ruff.yml create mode 100644 pyproject.toml diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml new file mode 100644 index 000000000..757cae68c --- /dev/null +++ b/.github/workflows/ruff.yml @@ -0,0 +1,8 @@ +name: Ruff +on: [ push, pull_request ] +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: chartboost/ruff-action@v1 diff --git a/README.md b/README.md index d439ccb10..e939a3cb6 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,13 @@ + # Retail Demo Store A sample retail web application and workshop platform intended as an educational tool for demonstrating how AWS infrastructure and services can be used to build compelling customer experiences for eCommerce, retail, and digital marketing use-cases. +# Build Status +[![Ruff](./actions/workflows/ruff.yml/badge.svg)](./actions/workflows/ruff.yml) + + + **This project is intended for educational purposes only and not for production use.** ![Retail Demo Store Home Page](./workshop/images/retaildemostore-home-devices.png) diff --git a/generators/datagenerator/amplitude.py b/generators/datagenerator/amplitude.py index 2e6b9103d..65e689b0e 100644 --- a/generators/datagenerator/amplitude.py +++ b/generators/datagenerator/amplitude.py @@ -1,10 +1,8 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: MIT-0 -import datagenerator import json import requests -import yaml # Amplitude event support # This follows the Amplitude V2 HTTP Bulk API spec, here: diff --git a/generators/datagenerator/file.py b/generators/datagenerator/file.py index 9d8c28d7c..2de4ce28f 100644 --- a/generators/datagenerator/file.py +++ b/generators/datagenerator/file.py @@ -21,5 +21,5 @@ def __repr__(self): output = f'{self.event},{self.timestamp},{self.user_id},{self.anonymous_id},{self.platform}' if len(self.traits) > 0: output += self.traits - output += f'\n' + output += '\n' return output \ No newline at end of file diff --git a/generators/datagenerator/funnel.py b/generators/datagenerator/funnel.py index d34b48782..f764fdb7a 100644 --- a/generators/datagenerator/funnel.py +++ b/generators/datagenerator/funnel.py @@ -4,9 +4,8 @@ import random import numpy as np import datetime -import inspect from datagenerator.output import OutputFormatter -from collections.abc import Mapping, Iterable +from collections.abc import Iterable class Funnel: def __init__(self, timestamp, funnel, user): diff --git a/generators/datagenerator/output.py b/generators/datagenerator/output.py index fd038e8cf..d220b0655 100644 --- a/generators/datagenerator/output.py +++ b/generators/datagenerator/output.py @@ -59,7 +59,7 @@ def to_amplitude(self, config, debug=False): batch.append(event) if len(batch) > 0: response = sender.send_batch(funnel.platform, batch, debug) - if response != None and response.status_code > 200: + if response is not None and response.status_code > 200: print(f'Error sending to Amplitude: {response.text}') print(f'Processed {count} funnels...') @@ -80,6 +80,6 @@ def to_segment(self, config_file, debug=False): batch.append(event) if len(batch) > 0: response = sender.send_batch(funnel.platform, batch, debug) - if response != None and response.status_code > 200: + if response is not None and response.status_code > 200: print(f'Error sending to Segment: {response.text}') print(f'Processed {count} funnels...') \ No newline at end of file diff --git a/generators/datagenerator/segment.py b/generators/datagenerator/segment.py index 2f6c3c032..e0068a403 100644 --- a/generators/datagenerator/segment.py +++ b/generators/datagenerator/segment.py @@ -87,7 +87,7 @@ def send_batch(self, platform, events, debug=False): } key = self.config_keys[platform] - if key != None: + if key is not None: events_str = json.dumps(batch_events, default=lambda x: x.__dict__) #print(f'Batch length bytes: {len(events_str)}') if debug: diff --git a/generators/datagenerator/users.py b/generators/datagenerator/users.py index 390a0befd..ec1a67d7b 100644 --- a/generators/datagenerator/users.py +++ b/generators/datagenerator/users.py @@ -2,12 +2,8 @@ # SPDX-License-Identifier: MIT-0 import random -import datetime -import uuid import json -import numpy as np import gzip -import codecs import bisect from faker import Faker from faker.providers import internet @@ -130,7 +126,7 @@ def new_file(cls, filename, num_users, class User: def __init__(self, category_preference_personas, selectable_user, id_string=None): - if(id_string != None): + if(id_string is not None): self.id = id_string else: self.id = str(random.randint(1000000000, 99999999999)) @@ -196,7 +192,7 @@ def __init__(self, category_preference_personas, selectable_user, id_string=None ] def set_traits(self, traits): - if traits != None: + if traits is not None: for (k,v) in traits.items(): self.traits[k] = random.choice(v) diff --git a/generators/generate_interactions_personalize.py b/generators/generate_interactions_personalize.py index 3acc5fa3b..6302617bd 100644 --- a/generators/generate_interactions_personalize.py +++ b/generators/generate_interactions_personalize.py @@ -114,8 +114,8 @@ def generate_user_items(out_users_filename, out_items_filename, in_users_filenam 'promoted': 'PROMOTED'}) # Since GENDER column requires a value for all rows, default all nulls to "Any" products_dataset_df['GENDER'].fillna(GENDER_ANY, inplace = True) - products_dataset_df.loc[products_dataset_df['PROMOTED'] == True, 'PROMOTED'] = 'Y' - products_dataset_df['PROMOTED'].fillna(NOT_PROMOTED, inplace = True) + products_dataset_df['PROMOTED'].fillna(False, inplace = True) + products_dataset_df['PROMOTED'] = products_dataset_df['PROMOTED'].replace({True: 'Y', False: 'N'}) products_dataset_df.to_csv(out_items_filename, index=False) users_dataset_df = users_df[['id', 'age', 'gender']] @@ -157,7 +157,8 @@ def generate_interactions(out_interactions_filename, users_df, products_df): average_product_price = int(products_df.price.mean()) print('Average product price: ${:.2f}'.format(average_product_price)) - if seconds_increment <= 0: raise AssertionError(f"Should never happen: {seconds_increment} <= 0") + if seconds_increment <= 0: + raise AssertionError(f"Should never happen: {seconds_increment} <= 0") print('Minimum interactions to generate: {}'.format(min_interactions)) print('Starting timestamp: {} ({})'.format(next_timestamp, @@ -275,7 +276,7 @@ def generate_interactions(out_interactions_filename, users_df, products_df): first_prod = user_category_to_first_prod[usercat_key] prods_subset_df = product_affinities_bycatgender[(category, gender)][first_prod] - if not usercat_key in user_category_to_first_prod: + if usercat_key not in user_category_to_first_prod: # If the user has not yet selected a first product for this category # we do it by choosing between all products for gender. @@ -296,7 +297,7 @@ def generate_interactions(out_interactions_filename, users_df, products_df): user_to_product[user['id']].add(product['id']) - if not usercat_key in user_category_to_first_prod: + if usercat_key not in user_category_to_first_prod: user_category_to_first_prod[usercat_key] = product['id'] # Decide if the product the user is interacting with is discounted diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..bea9e39a6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,48 @@ +[tool.ruff] +# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. +select = ["E", "F"] + +# bastil@ ignore Line too long E501 for now - gives too many warnings +ignore = ["E501"] + +# Allow autofix for all enabled rules (when `--fix`) is provided. +fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"] +unfixable = [] + +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".mypy_cache", + ".nox", + ".pants.d", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", +] + +# Same as Black. +line-length = 88 + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +# Assume Python 3.10 +target-version = "py310" + +[tool.ruff.mccabe] +# Unlike Flake8, default to a complexity level of 10. +max-complexity = 10 \ No newline at end of file diff --git a/src/aws-lambda/alexa-skill-lambda/alexa-skill-lambda.py b/src/aws-lambda/alexa-skill-lambda/alexa-skill-lambda.py index ac385cc3f..d8f646d1e 100644 --- a/src/aws-lambda/alexa-skill-lambda/alexa-skill-lambda.py +++ b/src/aws-lambda/alexa-skill-lambda/alexa-skill-lambda.py @@ -16,15 +16,12 @@ from ask_sdk_core.skill_builder import SkillBuilder from ask_sdk_core.dispatch_components import AbstractRequestHandler from ask_sdk_core.dispatch_components import AbstractExceptionHandler -from ask_sdk_core.handler_input import HandlerInput from ask_sdk_model.dialog import ElicitSlotDirective, DynamicEntitiesDirective, DelegateDirective from ask_sdk_model.dialog_state import DialogState from ask_sdk_model.er.dynamic import Entity, EntityValueAndSynonyms, EntityListItem, UpdateBehavior from ask_sdk_model.slu.entityresolution import StatusCode -from ask_sdk_model.interfaces.connections import SendRequestDirective -from ask_sdk_model.ui import AskForPermissionsConsentCard import boto3 import json @@ -117,7 +114,7 @@ def get_cognito_user_details(handler_input): logger.info(f"Got user info from Cognito: {user_details}") if 'custom:profile_user_id' not in user_details: - logger.warning(f"Profile user has not been selected for Cognito user") + logger.warning("Profile user has not been selected for Cognito user") raise Exception("Must use default user because simulation user not selected.") else: user_details['cognito_loaded'] = True @@ -210,7 +207,7 @@ def is_pinpoint_email_channel_enabled() -> bool: email_channel_response = pinpoint.get_email_channel(ApplicationId=PINPOINT_APP_ID) except ClientError as error: logger.info('Unable to find Email Channel configured for Pinpoint application: {}'.format(error)) - return False; + return False email_channel_from_address = None email_channel_enabled = False @@ -245,7 +242,7 @@ def send_order_confirm_email(handler_input, orders, add_images=True): # Specify content: subject = "Your order has been received!" heading = "Welcome," - subheading = f"Your order has been placed." + subheading = "Your order has been placed." intro_text = f"""We will meet you at your pump with the following order ({order_ids}):""" html_intro_text = intro_text.replace('\n', '

') @@ -897,12 +894,12 @@ def handle(self, handler_input): order_response = submit_order(handler_input) send_order_confirm_email(handler_input, [order_response], False) - speak_output += f"It will be ready when you arrive" + speak_output += "It will be ready when you arrive" if user_details['cognito_loaded']: name = user_details.get('custom:profile_first_name', '') speak_output += f" {name}" - speak_output += f". Hope to see you again soon." + speak_output += ". Hope to see you again soon." return ( handler_input.response_builder .speak(speak_output) diff --git a/src/aws-lambda/ivs-create-channels/ivs-create-channels.py b/src/aws-lambda/ivs-create-channels/ivs-create-channels.py index 3227850d3..162c14fcb 100644 --- a/src/aws-lambda/ivs-create-channels/ivs-create-channels.py +++ b/src/aws-lambda/ivs-create-channels/ivs-create-channels.py @@ -167,7 +167,7 @@ def create_ivs_channels(event, _): Type='String', Overwrite=True ) - except botocore.exceptions.EndpointConnectionError as ex: + except botocore.exceptions.EndpointConnectionError: logger.error("Could not create any IVS channels - probably because IVS is not supported in region. " f"Channel name: {channel_name}. Region: {ivs_client.meta.region_name}") @@ -209,7 +209,7 @@ def delete_all_channels(event, _): """ Deletes all IVS channels referenced in the SSM_VIDEO_CHANNEL_MAP_PARAM. """ - logger.info(f"Deleting all IVS channels in stack") + logger.info("Deleting all IVS channels in stack") if is_ssm_parameter_set(SSM_VIDEO_CHANNEL_MAP_PARAM): video_channel_param_value = ssm_client.get_parameter(Name=SSM_VIDEO_CHANNEL_MAP_PARAM)['Parameter']['Value'] video_channel_map = json.loads(video_channel_param_value) diff --git a/src/aws-lambda/location-geofence-event/location-geofence-event.py b/src/aws-lambda/location-geofence-event/location-geofence-event.py index 846a71b75..9731cf7d1 100644 --- a/src/aws-lambda/location-geofence-event/location-geofence-event.py +++ b/src/aws-lambda/location-geofence-event/location-geofence-event.py @@ -494,7 +494,7 @@ def send_pickup_sms(all_orders, add_order_details=False): Returns: Nothing but sends an SMS. """ - logger.info(f"Collecting phone numbers to send SMSs") + logger.info("Collecting phone numbers to send SMSs") phone_to_orders = defaultdict(list) for order in all_orders: @@ -543,7 +543,7 @@ def remove_browser_notification_connections(user_id, connection_ids): UpdateExpression='DELETE connectionIds :c', ExpressionAttributeValues=dynamo_update_expression ) - logger.info(f"Gone connections deleted") + logger.info("Gone connections deleted") def send_browser_notification(user_id, data): diff --git a/src/aws-lambda/personalize-pre-create-resources/delete_dataset_groups.py b/src/aws-lambda/personalize-pre-create-resources/delete_dataset_groups.py index 4a4c38f55..2b76399c3 100644 --- a/src/aws-lambda/personalize-pre-create-resources/delete_dataset_groups.py +++ b/src/aws-lambda/personalize-pre-create-resources/delete_dataset_groups.py @@ -338,7 +338,7 @@ def _delete_dataset_group(dataset_group_arn: str, wait_for_resources: bool = Tru logger.info('Waiting for dataset group to be deleted') time.sleep(20) else: - raise ResourcePending(f'Dataset group still being deleted') + raise ResourcePending('Dataset group still being deleted') def delete_dataset_groups(dataset_group_names: List[str], region: str = None, wait_for_resources: bool = True): min_botocore_version = '1.23.15' # As of re:Invent 2021 when domain recommenders were added to the API diff --git a/src/aws-lambda/pinpoint-auto-workshop/pinpoint-auto-workshop.py b/src/aws-lambda/pinpoint-auto-workshop/pinpoint-auto-workshop.py index 392e621d9..88c903f6d 100644 --- a/src/aws-lambda/pinpoint-auto-workshop/pinpoint-auto-workshop.py +++ b/src/aws-lambda/pinpoint-auto-workshop/pinpoint-auto-workshop.py @@ -88,10 +88,11 @@ def create_email_template(template_name, template_fname_root, subject, descripti TemplateName=template_name ) break - except pinpoint.exceptions.BadRequestException as e: + except pinpoint.exceptions.BadRequestException: try: - delete_response = pinpoint.delete_email_template(TemplateName=template_name) - except: + pinpoint.delete_email_template(TemplateName=template_name) + except BaseException as error: + logger.info('An exception occurred: {}'.format(error)) pass backoff_seconds = 30 logger.info(f"Waiting for old template to delete: {template_name} - waiting {backoff_seconds} seconds") @@ -133,10 +134,11 @@ def create_sms_template(template_name, body, description, recommender_id=None): TemplateName=template_name ) break - except pinpoint.exceptions.BadRequestException as e: + except pinpoint.exceptions.BadRequestException: try: - delete_response = pinpoint.delete_sms_template(TemplateName=template_name) - except: + pinpoint.delete_sms_template(TemplateName=template_name) + except BaseException as error: + logger.info('An exception occurred: {}'.format(error)) pass backoff_seconds = 30 logger.info(f"Waiting for old template to delete: {template_name} - waiting {backoff_seconds} seconds") @@ -275,7 +277,7 @@ def create_all_email_users_segment(application_id): Returns: Segment config. Returns even if already exists. """ - segment_name = f'AllEmailUsers' + segment_name = 'AllEmailUsers' segment_config = get_segment(application_id, segment_name) if not segment_config: diff --git a/src/aws-lambda/pinpoint-sms-alerts/pinpoint-sms-alerts.py b/src/aws-lambda/pinpoint-sms-alerts/pinpoint-sms-alerts.py index c35768512..9f0f294f5 100644 --- a/src/aws-lambda/pinpoint-sms-alerts/pinpoint-sms-alerts.py +++ b/src/aws-lambda/pinpoint-sms-alerts/pinpoint-sms-alerts.py @@ -13,11 +13,8 @@ import json import boto3 -import botocore import logging import os -from datetime import datetime, timedelta -from botocore.exceptions import ClientError logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/src/aws-lambda/segment-personalize-events-destination/segment-personalize-events-destination.py b/src/aws-lambda/segment-personalize-events-destination/segment-personalize-events-destination.py index 6062cc88d..725504314 100644 --- a/src/aws-lambda/segment-personalize-events-destination/segment-personalize-events-destination.py +++ b/src/aws-lambda/segment-personalize-events-destination/segment-personalize-events-destination.py @@ -20,7 +20,7 @@ # Initialize the Amazon Personalize events boto object personalize_events = boto3.client('personalize-events') -if not 'personalize_tracking_id' in os.environ or os.environ['personalize_tracking_id'] == '': +if 'personalize_tracking_id' not in os.environ or os.environ['personalize_tracking_id'] == '': logger.error("Missing personalize_tracking_id environment variable in lambda configuration.") raise Exception('personalize_tracking_id not configured as environment variable') else: @@ -37,7 +37,7 @@ def lambda_handler(event, context): # Make sure this event contains an itemId since this is required for the Retail Demo Store # dataset - you can also check for specific event names here if needed, and only pass the ones # that you want to use in the training dataset - if (not 'productId' in event['properties']): + if ('productId' not in event['properties']): logger.debug("Got event with no productId, discarding.") return @@ -78,12 +78,12 @@ def lambda_handler(event, context): logger.debug('put_events parameters: {}'.format(json.dumps(params, indent = 2))) # Call put_events - response = personalize_events.put_events(**params) + personalize_events.put_events(**params) else: logger.debug("Segment event does not contain required fields (anonymousId and sku)") - except ValueError as ve: + except ValueError: logger.error("Invalid JSON format received, check your event sources.") - except KeyError as ke: + except KeyError: logger.error("Invalid configuration for Personalize, most likely.") except ClientError as ce: logger.error("ClientError: ") diff --git a/src/aws-lambda/segment-personalize-inference-destination/segment-personalize-inference-destination.py b/src/aws-lambda/segment-personalize-inference-destination/segment-personalize-inference-destination.py index fa2c7b23d..7814a2edc 100644 --- a/src/aws-lambda/segment-personalize-inference-destination/segment-personalize-inference-destination.py +++ b/src/aws-lambda/segment-personalize-inference-destination/segment-personalize-inference-destination.py @@ -17,12 +17,12 @@ logger.setLevel(logging.INFO) # Check that the Segment env variables are set correctly -if not 'segment_personas_write_key' in os.environ or os.environ['segment_personas_write_key'] == '': +if 'segment_personas_write_key' not in os.environ or os.environ['segment_personas_write_key'] == '': raise Exception('segment_personas_write_key is null or not defined.') else: analytics.write_key = os.environ['segment_personas_write_key'] -if not 'recommendations_service_url' in os.environ or os.environ['recommendations_service_url'] == '': +if 'recommendations_service_url' not in os.environ or os.environ['recommendations_service_url'] == '': raise Exception('recommendations_service_url not configured as environment variable') else: recommendations_service_url = os.environ['recommendations_service_url'] @@ -45,7 +45,7 @@ def lambda_handler(event, context): logger.debug(recommendations) # Send the user recommendations to Segment analytics.identify(user_id, { 'personalized_recommendations': recommendations }) - except ValueError as ve: + except ValueError: logger.error("Invalid JSON format received, check your event sources.") - except KeyError as ke: + except KeyError: logger.error("Invalid configuration for Personalize, most likely.") diff --git a/src/carts/src/carts-service/dynamo_setup.py b/src/carts/src/carts-service/dynamo_setup.py index 2d476af27..8061586d4 100644 --- a/src/carts/src/carts-service/dynamo_setup.py +++ b/src/carts/src/carts-service/dynamo_setup.py @@ -4,8 +4,6 @@ import os import time import boto3 -import requests -import json from server import app def create_table(client, ddb_table_name, attribute_definitions, key_schema, global_secondary_indexes=None): diff --git a/src/carts/src/carts-service/services.py b/src/carts/src/carts-service/services.py index d749903d9..aaf24c638 100644 --- a/src/carts/src/carts-service/services.py +++ b/src/carts/src/carts-service/services.py @@ -12,7 +12,7 @@ from boto3.dynamodb.types import TypeSerializer, TypeDeserializer from decimal import Decimal -from werkzeug.exceptions import BadRequest, NotFound +from werkzeug.exceptions import BadRequest class CartService: diff --git a/src/carts/test/integ/test_carts.py b/src/carts/test/integ/test_carts.py index fc32d1207..c0ff1d905 100644 --- a/src/carts/test/integ/test_carts.py +++ b/src/carts/test/integ/test_carts.py @@ -71,7 +71,7 @@ def test_update_with_id_mismatch(): assert_that(response.json()["error"]).is_equal_to("Bad request, please check your input") def test_update_with_nonexistent_cart(): - endpoint = f"/carts/nonexistent_id" + endpoint = "/carts/nonexistent_id" body = {"id": "invalid_value"} response = requests.put(full_request_url(carts_api_url, endpoint), data=json.dumps(body)) assert_that(response.status_code).is_equal_to(404) diff --git a/src/location/src/location-service/app.py b/src/location/src/location-service/app.py index a12ff205e..d1def8ee2 100644 --- a/src/location/src/location-service/app.py +++ b/src/location/src/location-service/app.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: MIT-0 from flask import Flask -from flask import abort, jsonify, request +from flask import jsonify from flask_cors import CORS import boto3 diff --git a/src/offers/src/offers-service/app.py b/src/offers/src/offers-service/app.py index 314d24da3..800fd3e02 100644 --- a/src/offers/src/offers-service/app.py +++ b/src/offers/src/offers-service/app.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: MIT-0 from flask import Flask -from flask import abort, jsonify, request +from flask import abort, jsonify from flask_cors import CORS import json diff --git a/src/products/load_catalog.py b/src/products/load_catalog.py index 081250d52..215d7569a 100644 --- a/src/products/load_catalog.py +++ b/src/products/load_catalog.py @@ -35,7 +35,6 @@ import sys import getopt import time -import requests import yaml from boto3 import resource from decimal import Decimal @@ -108,7 +107,7 @@ def verify_local_ddb_running(endpoint,dynamodb): dynamodb.tables.all() print("DynamoDB local is responding!") return - except Exception as e: + except Exception: print("Local DynamoDB service is not ready yet... pausing before trying again") time.sleep(2) print("Local DynamoDB service not responding; verify that your docker-compose .env file is setup correctly") diff --git a/src/recommendations/src/recommendations-service/app.py b/src/recommendations/src/recommendations-service/app.py index ab064ae95..15081a3d9 100644 --- a/src/recommendations/src/recommendations-service/app.py +++ b/src/recommendations/src/recommendations-service/app.py @@ -6,7 +6,6 @@ from aws_xray_sdk.ext.flask.middleware import XRayMiddleware from aws_xray_sdk.core import patch_all -patch_all() from typing import Dict, List, Tuple, Union from flask import Flask, jsonify, Response @@ -28,6 +27,9 @@ import logging from datetime import datetime +# X-ray setup +patch_all() + NUM_DISCOUNTS = 2 EXPERIMENTATION_LOGGING = True @@ -366,8 +368,10 @@ def related(): # The default filter includes products from the same category as the current item. filter_ssm = request.args.get('filter', filter_include_categories_param_name) # We have short names for these filters - if filter_ssm == 'cstore': filter_ssm = filter_cstore_param_name - elif filter_ssm == 'purchased': filter_ssm = filter_purchased_param_name + if filter_ssm == 'cstore': + filter_ssm = filter_cstore_param_name + elif filter_ssm == 'purchased': + filter_ssm = filter_purchased_param_name app.logger.info("Filter SSM for /related: %s", filter_ssm) filter_values = None @@ -448,8 +452,10 @@ def recommendations(): # The default filter is the not-already-purchased filter filter_ssm = request.args.get('filter', filter_purchased_param_name) # We have short names for these filters - if filter_ssm == 'cstore': filter_ssm = filter_cstore_param_name - elif filter_ssm == 'purchased': filter_ssm = filter_purchased_param_name + if filter_ssm == 'cstore': + filter_ssm = filter_cstore_param_name + elif filter_ssm == 'purchased': + filter_ssm = filter_purchased_param_name app.logger.info(f"Filter SSM for /recommendations: {filter_ssm}") fully_qualify_image_urls = request.args.get('fullyQualifyImageUrls', '0').lower() in [ 'true', 't', '1'] @@ -514,8 +520,10 @@ def popular(): # The default filter is the exclude already purchased and c-store products filter filter_ssm = request.args.get('filter', filter_purchased_cstore_param_name) # We have short names for these filters - if filter_ssm == 'cstore': filter_ssm = filter_cstore_param_name - elif filter_ssm == 'purchased': filter_ssm = filter_purchased_cstore_param_name + if filter_ssm == 'cstore': + filter_ssm = filter_cstore_param_name + elif filter_ssm == 'purchased': + filter_ssm = filter_purchased_cstore_param_name app.logger.info(f"Filter SSM for /recommendations: {filter_ssm}") fully_qualify_image_urls = request.args.get('fullyQualifyImageUrls', '0').lower() in [ 'true', 't', '1'] diff --git a/src/recommendations/src/recommendations-service/experimentation/resolvers.py b/src/recommendations/src/recommendations-service/experimentation/resolvers.py index 4947a2fdf..09b081e61 100644 --- a/src/recommendations/src/recommendations-service/experimentation/resolvers.py +++ b/src/recommendations/src/recommendations-service/experimentation/resolvers.py @@ -5,7 +5,6 @@ import requests import boto3 -import json import urllib.parse import logging @@ -224,7 +223,8 @@ def get_items(self, **kwargs): if kwargs.get('promotion'): params['promotions'] = [ kwargs['promotion'] ] - if 'context' in kwargs and kwargs['context'] is not None: params['context'] = kwargs['context'] + if 'context' in kwargs and kwargs['context'] is not None: + params['context'] = kwargs['context'] if item_id: params['itemId'] = item_id @@ -351,7 +351,8 @@ def get_items(self, **kwargs): if params.get('filterArn') and kwargs.get('filter_values'): params['filterValues'] = kwargs.get('filter_values') - if 'context' in kwargs and kwargs['context'] is not None: params['context'] = kwargs['context'] + if 'context' in kwargs and kwargs['context'] is not None: + params['context'] = kwargs['context'] log.debug('PersonalizeRankingResolver - getting personalized ranking %s', params) diff --git a/src/recommendations/src/recommendations-service/experimentation/test_experiment.py b/src/recommendations/src/recommendations-service/experimentation/test_experiment.py index 55350cc91..80f7ba64a 100644 --- a/src/recommendations/src/recommendations-service/experimentation/test_experiment.py +++ b/src/recommendations/src/recommendations-service/experimentation/test_experiment.py @@ -5,7 +5,6 @@ import uuid import json -from unittest.mock import patch from experimentation.resolvers import ResolverFactory, PersonalizeRecommendationsResolver, DefaultProductResolver from experimentation.experiment_ab import ABExperiment from experimentation.experiment_interleaving import InterleavingExperiment diff --git a/src/recommendations/src/recommendations-service/experimentation/test_resolvers.py b/src/recommendations/src/recommendations-service/experimentation/test_resolvers.py index 40c45fe50..57feaddfe 100644 --- a/src/recommendations/src/recommendations-service/experimentation/test_resolvers.py +++ b/src/recommendations/src/recommendations-service/experimentation/test_resolvers.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: MIT-0 import unittest -import json import botocore import logging diff --git a/src/recommendations/src/recommendations-service/run_tests.py b/src/recommendations/src/recommendations-service/run_tests.py index 5e67e6a31..b49c60055 100644 --- a/src/recommendations/src/recommendations-service/run_tests.py +++ b/src/recommendations/src/recommendations-service/run_tests.py @@ -1,7 +1,9 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: MIT-0 -import os, sys, unittest +import os +import sys +import unittest os.environ['EVIDENTLY_PROJECT_NAME'] = 'retaildemostore' diff --git a/src/recommendations/test/integ/test_discounted.py b/src/recommendations/test/integ/test_discounted.py index 65e2ae16b..6c8403a28 100644 --- a/src/recommendations/test/integ/test_discounted.py +++ b/src/recommendations/test/integ/test_discounted.py @@ -1,4 +1,3 @@ -import sys import os from assertpy import assert_that from testhelpers.integ import ( diff --git a/src/search/src/search-service/app.py b/src/search/src/search-service/app.py index 811c6b8cd..751636dfa 100644 --- a/src/search/src/search-service/app.py +++ b/src/search/src/search-service/app.py @@ -5,8 +5,6 @@ from aws_xray_sdk.ext.flask.middleware import XRayMiddleware from aws_xray_sdk.core import patch_all -patch_all() - from flask import Flask, jsonify from flask import request from flask_cors import CORS @@ -16,6 +14,9 @@ import os import pprint +patch_all() + + INDEX_DOES_NOT_EXIST = 'index_not_found_exception' search_domain_scheme = os.environ.get('OPENSEARCH_DOMAIN_SCHEME', 'https') diff --git a/src/users/test/integ/test-users.py b/src/users/test/integ/test-users.py index 793a4b4cd..92d96c79b 100644 --- a/src/users/test/integ/test-users.py +++ b/src/users/test/integ/test-users.py @@ -1,5 +1,4 @@ import testhelpers.integ as integhelpers -import time import requests import os from dotenv import load_dotenv diff --git a/src/videos/src/videos-service/app.py b/src/videos/src/videos-service/app.py index 2fd230f9f..d5b6ea8b1 100644 --- a/src/videos/src/videos-service/app.py +++ b/src/videos/src/videos-service/app.py @@ -6,10 +6,6 @@ from aws_xray_sdk.ext.flask.middleware import XRayMiddleware from aws_xray_sdk.core import patch_all -patch_all() - -xray_recorder.begin_segment("Videos-init") - import logging import json import os @@ -25,6 +21,11 @@ from flask_cors import CORS +patch_all() + +xray_recorder.begin_segment("Videos-init") + + # -- Environment variables - defined by CloudFormation when deployed VIDEO_BUCKET = os.environ.get('RESOURCE_BUCKET') IMAGE_ROOT_URL = os.environ.get('IMAGE_ROOT_URL') + 'videos/' @@ -114,7 +115,7 @@ def get_featured_products(video_filepath, channel_id): """ subtitle_path = pathlib.Path(video_filepath).with_suffix('.srt') get_subs_command = get_ffmpeg_subs_cmd(video_filepath, subtitle_path) - process = subprocess.run( + subprocess.run( get_subs_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, shell=True) with open(subtitle_path) as f: subtitle_content = srt.parse(f) @@ -247,13 +248,15 @@ def stream(s3_video_key, ivs_channel_arn, channel_id): try: int(next(lines).strip()) time_range = next(lines).strip() - if not '-->' in time_range: + if "-->" not in time_range: raise ValueError(f'Expected a time range instead of {time_range}') send_text = '' while True: text = next(lines).strip() - if len(text) == 0: break - if len(send_text)>0: send_text+='\n' + if len(text) == 0: + break + if len(send_text)>0: + send_text+='\n' send_text += text put_ivs_metadata(ivs_channel_arn, send_text) except StopIteration: diff --git a/src/videos/src/videos-service/run_tests.py b/src/videos/src/videos-service/run_tests.py index 23957e196..aa2a4cd6f 100644 --- a/src/videos/src/videos-service/run_tests.py +++ b/src/videos/src/videos-service/run_tests.py @@ -1,7 +1,9 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: MIT-0 -import os, sys, unittest +import os +import sys +import unittest sys.argv += ['discover', os.path.dirname(sys.argv[0]), 'test_*.py']