Skip to content

Commit

Permalink
Refactor order service from go to python (#488)
Browse files Browse the repository at this point in the history
* resolving conflict

* refactor orders service to python

* add persistence to orders service with dynamodb

* orders table cloudformation output

* node-modules package-lock

* add validation and more testing

* search for specific table during local and add more test

* explicit routes and handlers

* update valid keys

* remove unused imports and scripts

* update valid keys for order service

* remove old go scripts and make order keys consistent

* remove ttl order inherits from cart

* Update app.py and remove unused imports

* fix linter errors

* fix linter errors-too long lines

* fix too long lines
  • Loading branch information
Adibuer-lab authored Sep 14, 2023
1 parent 275ba37 commit e8080b3
Show file tree
Hide file tree
Showing 30 changed files with 660 additions and 382 deletions.
4 changes: 4 additions & 0 deletions aws/cloudformation-templates/base/_template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,10 @@ Outputs:
Description: DynamoDB Table for Carts
Value: !GetAtt Tables.Outputs.CartsTable

OrdersTable:
Description: DynamoDB Table for Orders
Value: !GetAtt Tables.Outputs.OrdersTable

ExperimentStrategyTable:
Description: DynamoDB Table for Experiments
Value: !GetAtt Tables.Outputs.ExperimentStrategyTable
Expand Down
24 changes: 24 additions & 0 deletions aws/cloudformation-templates/base/tables.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,26 @@ Resources:
AttributeName: "ttl"
Enabled: true

OrdersTable:
Type: AWS::DynamoDB::Table
Properties:
AttributeDefinitions:
- AttributeName: "id"
AttributeType: "S"
- AttributeName: "username"
AttributeType: "S"
KeySchema:
- AttributeName: "id"
KeyType: "HASH"
BillingMode: "PAY_PER_REQUEST"
GlobalSecondaryIndexes:
- IndexName: username-index
KeySchema:
- AttributeName: "username"
KeyType: "HASH"
Projection:
ProjectionType: ALL

ExperimentStrategyTable:
Type: AWS::DynamoDB::Table
DependsOn: "ProductsTable"
Expand Down Expand Up @@ -124,6 +144,10 @@ Outputs:
Description: DynamoDB Table for Carts
Value: !Ref CartsTable

OrdersTable:
Description: DynamoDB Table for Carts
Value: !Ref OrdersTable

ExperimentStrategyTable:
Description: DynamoDB Table for Experiment Strategies
Value: !Ref ExperimentStrategyTable
7 changes: 5 additions & 2 deletions aws/cloudformation-templates/services/_template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,10 @@ Parameters:
Type: String
Default: none

OrdersTable:
Type: String
Default: none

ExperimentStrategyTable:
Type: String
Default: none
Expand Down Expand Up @@ -170,7 +174,6 @@ Resources:
ServiceDiscoveryNamespace: !Ref ServiceDiscoveryNamespace
ProductsTable: !Ref ProductsTable
CategoriesTable: !Ref CategoriesTable
CartsTable: !Ref CartsTable
ExperimentStrategyTable: !Ref ExperimentStrategyTable
ParameterPersonalizeEventTrackerId: !Ref ParameterPersonalizeEventTrackerId
ParameterAmplitudeApiKey: !Ref ParameterAmplitudeApiKey
Expand Down Expand Up @@ -283,6 +286,7 @@ Resources:
ParameterOptimizelySdkKey: !Ref ParameterOptimizelySdkKey
CleanupBucketLambdaArn: !Ref CleanupBucketLambdaArn
DeleteRepositoryLambdaArn: !GetAtt DeleteRepositoryLambdaFunction.Arn
OrdersTable: !Ref OrdersTable
WebRootUrl: !Ref WebRootUrl
ImageRootUrl: !Ref ImageRootUrl
Uid: !Sub ${ParentStackName}-${AWS::Region}
Expand Down Expand Up @@ -419,7 +423,6 @@ Resources:
ServiceDiscoveryNamespace: !Ref ServiceDiscoveryNamespace
ProductsTable: !Ref ProductsTable
CategoriesTable: !Ref CategoriesTable
CartsTable: !Ref CartsTable
ExperimentStrategyTable: !Ref ExperimentStrategyTable
ParameterPersonalizeEventTrackerId: !Ref ParameterPersonalizeEventTrackerId
ParameterAmplitudeApiKey: !Ref ParameterAmplitudeApiKey
Expand Down
5 changes: 5 additions & 0 deletions aws/cloudformation-templates/services/service/_template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,10 @@ Parameters:
Type: String
Default: none

OrdersTable:
Type: String
Default: none

ExperimentStrategyTable:
Type: String
Default: none
Expand Down Expand Up @@ -308,6 +312,7 @@ Resources:
ProductsTable: !Ref ProductsTable
CategoriesTable: !Ref CategoriesTable
CartsTable: !Ref CartsTable
OrdersTable: !Ref OrdersTable
ExperimentStrategyTable: !Ref ExperimentStrategyTable
PinpointAppId: !Ref PinpointAppId
WebRootUrl: !Ref WebRootUrl
Expand Down
8 changes: 8 additions & 0 deletions aws/cloudformation-templates/services/service/service.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,10 @@ Parameters:
Type: String
Default: none

OrdersTable:
Type: String
Default: none

ExperimentStrategyTable:
Type: String
Default: none
Expand Down Expand Up @@ -180,10 +184,12 @@ Resources:
- !Sub 'arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${ProductsTable}'
- !Sub 'arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${CategoriesTable}'
- !Sub 'arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${CartsTable}'
- !Sub 'arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${OrdersTable}'
- !Sub 'arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${ExperimentStrategyTable}'
- !Sub 'arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${ProductsTable}/index/*'
- !Sub 'arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${CategoriesTable}/index/*'
- !Sub 'arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${CartsTable}/index/*'
- !Sub 'arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${OrdersTable}/index/*'
- !Sub 'arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${ExperimentStrategyTable}/index/*'
- PolicyName: PinpointSMS
PolicyDocument:
Expand Down Expand Up @@ -389,6 +395,8 @@ Resources:
Value: !Ref CategoriesTable
- Name: DDB_TABLE_CARTS
Value: !Ref CartsTable
- Name: DDB_TABLE_ORDERS
Value: !Ref OrdersTable
- Name: WEB_ROOT_URL
Value: !Ref WebRootUrl
- Name: IMAGE_ROOT_URL
Expand Down
5 changes: 5 additions & 0 deletions aws/cloudformation-templates/template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -551,6 +551,7 @@ Resources:
ProductsTable: !GetAtt Base.Outputs.ProductsTable
CategoriesTable: !GetAtt Base.Outputs.CategoriesTable
CartsTable: !GetAtt Base.Outputs.CartsTable
OrdersTable: !GetAtt Base.Outputs.OrdersTable
ExperimentStrategyTable: !GetAtt Base.Outputs.ExperimentStrategyTable
ParameterPersonalizeEventTrackerId: !GetAtt Base.Outputs.ParameterPersonalizeEventTrackerId
ParameterAmplitudeApiKey: !GetAtt Base.Outputs.ParameterAmplitudeApiKey
Expand Down Expand Up @@ -923,6 +924,10 @@ Outputs:
CARTS_SERVICE_HOST="${ApiGateway.Outputs.APIEndpoint}"
DDB_TABLE_CARTS=${Base.Outputs.CartsTable}
# Orders service
ORDERS_SERVICE_HOST="${ApiGateway.Outputs.APIEndpoint}"
DDB_TABLE_ORDERS=${Base.Outputs.OrdersTable}
# Search service
OPENSEARCH_DOMAIN_SCHEME=https
OPENSEARCH_DOMAIN_HOST="${Base.Outputs.OpenSearchDomainEndpoint}"
Expand Down
5 changes: 3 additions & 2 deletions src/.env.template
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ DDB_ENDPOINT_OVERRIDE=http://ddb:3001
DDB_TABLE_PRODUCTS=products
DDB_TABLE_CATEGORIES=categories
DDB_TABLE_CARTS=carts
DDB_TABLE_ORDERS=orders
# Root URL to use when building fully qualified URLs to product detail view
WEB_ROOT_URL=http://localhost:8080
# Image root URL to use when building fully qualified URLs to product images
Expand Down Expand Up @@ -73,11 +74,11 @@ TEST_AGE_RANGE="25-34"

# orders service
ORDERS_API_URL=http://localhost:8004
TEST_ORDER_ID=1
TEST_ORDER_ID='209673d6-46a5-11ee-be56-0242ac120002'
TEST_USERNAME="user1344"

# recommendation service
RECOMMENDATIONS_API_URL="http://localhost:8005"

# carts service
CARTS_API_URL="http://localhost:8003"
CARTS_API_URL="http://localhost:8003"
3 changes: 2 additions & 1 deletion src/carts/src/carts-service/dynamo_setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ def verify_local_ddb_running(endpoint, dynamo_client):
for _ in range(5):
try:
response = dynamo_client.list_tables()
if response['TableNames'] == []:
#if does not contain ddb_table_carts, then create table
if ddb_table_carts not in response['TableNames'] :
create_table(
ddb_table_name=ddb_table_carts,
client=dynamo_client,
Expand Down
7 changes: 5 additions & 2 deletions src/carts/src/carts-service/services.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ class CartService:
serializer = TypeSerializer()
deserializer = TypeDeserializer()

ALLOWED_KEYS = {'id', 'items', 'ttl', 'username'}

@staticmethod
def deserialize_item(item):
"""
Expand Down Expand Up @@ -56,8 +58,9 @@ def validate_cart(cart):
Args:
cart: The cart to be validated.
"""
allowed_keys = ['id', 'items', 'ttl', 'username']
if not all(key in allowed_keys for key in cart.keys()):
invalid_keys = set(cart.keys()) - CartService.ALLOWED_KEYS
if invalid_keys:
app.logger.info(f"Invalid keys in cart: {invalid_keys}")
raise BadRequest


Expand Down
9 changes: 9 additions & 0 deletions src/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,15 @@ services:

orders:
container_name: orders
depends_on:
- ddb
environment:
- AWS_REGION
- AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY
- AWS_SESSION_TOKEN
- DDB_TABLE_ORDERS
- DDB_ENDPOINT_OVERRIDE
build:
context: ./orders
networks:
Expand Down
25 changes: 18 additions & 7 deletions src/orders/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,21 @@
FROM public.ecr.aws/s5z3t2n9/golang:1.11-alpine AS build
FROM public.ecr.aws/docker/library/python:3.10-slim-bullseye

WORKDIR /src/
COPY src/orders-service/*.* /src/
RUN apk add --no-cache git
RUN CGO_ENABLED=0 go build -o /bin/orders-service

FROM scratch
COPY --from=build /bin/orders-service /bin/orders-service
RUN apt-get update && apt-get install -y g++

COPY src/orders-service/requirements.txt .

RUN python3 -m pip install -r requirements.txt

COPY src/orders-service/server.py .
COPY src/orders-service/app.py .
COPY src/orders-service/routes.py .
COPY src/orders-service/services.py .
COPY src/orders-service/handlers.py .
COPY src/orders-service/dynamo_setup.py .

EXPOSE 80
ENTRYPOINT ["/bin/orders-service"]

ENTRYPOINT ["python"]
CMD ["app.py"]
2 changes: 1 addition & 1 deletion src/orders/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ When deployed to AWS, CodePipeline is used to build and deploy the Orders servic
The Orders service can be built and run locally (in Docker) using Docker Compose. See the [local development instructions](../) for details. **From the `../src` directory**, run the following command to build and deploy the service locally.

```console
foo@bar:~$ docker compose up --build orders
foo@bar:~$ docker-compose up --build orders
```

Once the container is up and running, you can access it in your browser or with a utility such as [Postman](https://www.postman.com/) at [http://localhost:8004](http://localhost:8004).
Expand Down
25 changes: 25 additions & 0 deletions src/orders/src/orders-service/app.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: MIT-0

import logging
import os
from server import app
from handlers import handler_bp
from routes import routes_bp


# Set up logging
logging.basicConfig(level=logging.INFO, handlers=[logging.StreamHandler()])
app.register_blueprint(handler_bp)
app.register_blueprint(routes_bp)
# Log a message at the start of the script
app.logger.info('Starting app.py')

if __name__ == '__main__':
try:
port = os.getenv('PORT', '80')
app.run(host='0.0.0.0', port=int(port))
except Exception as e:
# Log the error message and the type of the exception
app.logger.error(f'Error starting server: {str(e)}')
app.logger.error(f'Type of error: {type(e)}')
101 changes: 101 additions & 0 deletions src/orders/src/orders-service/dynamo_setup.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: MIT-0

import os
import time
import boto3
from server import app

def create_table(client, ddb_table_name,
attribute_definitions,
key_schema,
global_secondary_indexes=None):
try:
client.create_table(
TableName=ddb_table_name,
KeySchema=key_schema,
AttributeDefinitions=attribute_definitions,
GlobalSecondaryIndexes=global_secondary_indexes or [],
BillingMode="PAY_PER_REQUEST",
)
print(f'Created table: {ddb_table_name}')
except Exception as e:
if e.response["Error"]["Code"] == "ResourceInUseException":
app.logger.info(f'Table {ddb_table_name} already exists; continuing...')
else:
raise e


# DynamoDB table names passed via environment
ddb_table_orders = os.getenv("DDB_TABLE_ORDERS")

# Allow DDB endpoint to be overridden to support amazon/dynamodb-local
ddb_endpoint_override = os.getenv("DDB_ENDPOINT_OVERRIDE")
running_local = False

dynamo_client = None

def verify_local_ddb_running(endpoint, dynamo_client):
app.logger.info(f"Verifying that local DynamoDB is running at: {endpoint}")
for _ in range(5):
try:
response = dynamo_client.list_tables()
if ddb_table_orders not in response["TableNames"]:
create_table(
ddb_table_name=ddb_table_orders,
client=dynamo_client,
attribute_definitions=[
{"AttributeName": "id", "AttributeType": "S"},
{"AttributeName": "username", "AttributeType": "S"}
],
key_schema=[
{"AttributeName": "id", "KeyType": "HASH"},
],
global_secondary_indexes=[
{
"IndexName": "username-index",
"KeySchema": [{
"AttributeName": "username",
"KeyType": "HASH"}],
"Projection": {"ProjectionType": "ALL"},
"ProvisionedThroughput": {
"ReadCapacityUnits": 5,
"WriteCapacityUnits": 5,
}
}
]
)
app.logger.info("DynamoDB local is responding!")
return
except Exception as e:
app.logger.info(e)
app.logger.info(
"Local DynamoDB service is not ready yet... pausing before trying again"
)
time.sleep(2)
app.logger.error(
"Local DynamoDB service not responding;\
verify that your docker-compose .env file is setup correctly"
)
exit(1)

def setup():
global dynamo_client, running_local

if ddb_endpoint_override:
running_local = True
app.logger.info("Creating DDB client with endpoint override: "
+ ddb_endpoint_override)
dynamo_client = boto3.client(
'dynamodb',
endpoint_url=ddb_endpoint_override,
region_name='us-west-2',
aws_access_key_id='XXXX',
aws_secret_access_key='XXXX'
)
verify_local_ddb_running(ddb_endpoint_override, dynamo_client)
else:
running_local = False
dynamo_client = boto3.client('dynamodb')

setup()
Loading

0 comments on commit e8080b3

Please sign in to comment.