Skip to content

Commit

Permalink
Usability improvements.
Browse files Browse the repository at this point in the history
  • Loading branch information
thvasilo committed Aug 8, 2020
1 parent 987c03e commit 9659b7a
Show file tree
Hide file tree
Showing 10 changed files with 608 additions and 136 deletions.
20 changes: 18 additions & 2 deletions NOTICE
Original file line number Diff line number Diff line change
@@ -1,2 +1,18 @@
Fraud Detection Using Machine Learning
Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
Fraud Detection using Machine Learing
Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.

**********************
THIRD PARTY COMPONENTS
**********************
This software includes third party software subject to the following copyrights:

Boto 3 - The AWS SDK for Python under the Apache 2.0 License
requests - A simple, yet elegant HTTP library under the Apache 2.0 License
aws-requests-auth under the BSD 3-Clause License
pandas under the BSD 3-Clause License
numpy under the BSD 3-Clause License
scipy under the BSD 3-Clause License
seaborn under the BSD 3-Clause License
scikit-learn under the BSD 3-Clause License
imbalanced-learn under the MIT License
torch under the BSD 3-Clause License
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ To get started quickly, use the following quick-launch link to launch a CloudFor

| Region | Stack |
| ---- | ---- |
|US West (Oregon) | [<img src="https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png">](https://us-west-2.console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/review?templateURL=https://s3.amazonaws.com/sagemaker-solutions-us-west-2/Fraud-detection-using-machine-learning/deployment/fraud-detection-using-machine-learning.yaml&stackName=SageMaker-Fraud-Machine-Learning) |
|US West (Oregon) | [<img src="https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png">](https://us-west-2.console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/review?templateURL=https://sagemaker-solutions-us-west-2.s3-us-west-2.amazonaws.com/Fraud-detection-using-machine-learning/build/packaged.yaml&stackName=SageMaker-Fraud-Machine-Learning) |


### Additional Instructions
Expand Down
127 changes: 72 additions & 55 deletions deployment/fraud-detection-using-machine-learning.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,54 +3,43 @@ Description: >-
(SO0056) - fraud-detection-using-machine-learning: Solution for predicting
fraud events with ML using Amazon SageMaker. Version 3
Parameters:
ModelDataBucketName:
StackVersion:
Description: |
CloudFormation Stack version.
Use 'release' version unless you are customizing the
CloudFormation templates and solution artifacts.
Type: String
Description: New bucket for storing the Amazon SageMaker model and training data.
OutputBucketName:
Type: String
Description: Bucket to store the predictions to be visualized using Quicksight.
Default: release
AllowedValues:
- release
- development
KinesisFirehosePrefix:
Type: String
Default: fraud-detection/firehose/
Description: Kinesis Firehose prefix for delivery of processed events.
SolutionsS3BucketName:
Description: Enter the name of the S3 bucket for the solution
Type: String
Default: "sagemaker-solutions"
Metadata:
'AWS::CloudFormation::Interface':
ParameterGroups:
- Label:
default: Amazon S3 Bucket Configuration
Parameters:
- ModelDataBucketName
- OutputBucketName
- Label:
default: Amazon Kinesis Firehose Configuration
Parameters:
- KinesisFirehosePrefix
- Label:
default: Solution Configuration Parameters
Parameters:
- SolutionsS3BucketName
ParameterLabels:
SolutionsS3BucketName:
default: SageMaker Solution Bucket Base Name
ModelDataBucketName:
default: Model and Data Bucket Name
OutputBucketName:
default: Prediction Output Bucket Name
KinesisFirehosePrefix:
default: Kinesis Firehose S3 Prefix
Mappings:
Function:
FraudDetection:
S3Key: Fraud-detection-using-machine-learning/build/model_invocation.zip
SolutionsS3BucketName:
development:
Prefix: sagemaker-solutions
release:
Prefix: sagemaker-solutions
Resources:
ModelDataBucket:
Type: 'AWS::S3::Bucket'
Properties:
BucketName: !Ref ModelDataBucketName
PublicAccessBlockConfiguration:
BlockPublicAcls: true
BlockPublicPolicy: true
Expand All @@ -72,10 +61,9 @@ Resources:
bucket to store logs
- id: W51
reason: Default policy works fine
S3Bucket2:
OutputBucket:
Type: 'AWS::S3::Bucket'
Properties:
BucketName: !Ref OutputBucketName
PublicAccessBlockConfiguration:
BlockPublicAcls: true
BlockPublicPolicy: true
Expand Down Expand Up @@ -171,22 +159,29 @@ Resources:
Properties:
OnCreate:
- Content:
Fn::Base64: !Sub |
set -e
# perform following actions as ec2-user
sudo -u ec2-user -i <<EOF
cd /home/ec2-user/SageMaker
# copy source files
aws s3 sync s3://${SolutionsS3BucketName}-${AWS::Region}/Fraud-detection-using-machine-learning/ .
# set environment variables via .env file
touch .env
echo "AWS_ACCOUNT_ID=${AWS::AccountId}" >> .env
echo "AWS_REGION=${AWS::Region}" >> .env
echo "SAGEMAKER_IAM_ROLE=${NotebookInstanceExecutionRole.Arn}" >> .env
echo "SOLUTIONS_S3_BUCKET=${SolutionsS3BucketName}-${AWS::Region}" >> .env
echo "MODEL_DATA_S3_BUCKET=${ModelDataBucketName}" >> .env
echo "REST_API_GATEWAY=${RESTAPIGateway}" >> .env
EOF
Fn::Base64: !Sub
- |
set -e
# perform following actions as ec2-user
sudo -u ec2-user -i <<EOF
cd /home/ec2-user/SageMaker
# copy source files
aws s3 sync s3://${SolutionsS3BucketNamePrefix}-${AWS::Region}/Fraud-detection-using-machine-learning/ .
# copy data and unzip
aws s3 cp s3://sagemaker-e2e-solutions/fraud-detection/creditcardfraud.zip .
unzip creditcardfraud.zip -d ./source/notebooks/
# set environment variables via .env file
touch .env
echo "FRAUD_STACK_NAME=${AWS::StackName}" >> .env
echo "AWS_ACCOUNT_ID=${AWS::AccountId}" >> .env
echo "AWS_REGION=${AWS::Region}" >> .env
echo "SAGEMAKER_IAM_ROLE=${NotebookInstanceExecutionRole.Arn}" >> .env
echo "SOLUTIONS_S3_BUCKET=${SolutionsS3BucketNamePrefix}-${AWS::Region}" >> .env
echo "MODEL_DATA_S3_BUCKET=${ModelDataBucket}" >> .env
echo "REST_API_GATEWAY=${RESTAPIGateway}" >> .env
EOF
- SolutionsS3BucketNamePrefix:
Fn::FindInMap: [SolutionsS3BucketName, Ref: StackVersion, Prefix]
OnStart:
- Content:
Fn::Base64: !Sub |
Expand Down Expand Up @@ -227,13 +222,19 @@ Resources:
Action:
- s3:ListBucket
Resource:
- !Sub "arn:aws:s3:::${SolutionsS3BucketName}-${AWS::Region}"
- !Sub
- "arn:aws:s3:::${SolutionsS3BucketNamePrefix}-${AWS::Region}"
- SolutionsS3BucketNamePrefix:
Fn::FindInMap: [SolutionsS3BucketName, Ref: StackVersion, Prefix]
# Can get objects from the SageMaker Solutions Amazon S3 Bucket
- Effect: Allow
Action:
- s3:GetObject
Resource:
- !Sub "arn:aws:s3:::${SolutionsS3BucketName}-${AWS::Region}/*"
- !Sub
- "arn:aws:s3:::${SolutionsS3BucketNamePrefix}-${AWS::Region}/*"
- SolutionsS3BucketNamePrefix:
Fn::FindInMap: [SolutionsS3BucketName, Ref: StackVersion, Prefix]
- Effect: Allow
Action:
- 's3:GetBucketLocation'
Expand All @@ -245,11 +246,11 @@ Resources:
- !Join
- ''
- - 'arn:aws:s3:::'
- !Ref ModelDataBucketName
- !Ref ModelDataBucket
- !Join
- ''
- - 'arn:aws:s3:::'
- !Ref ModelDataBucketName
- !Ref ModelDataBucket
- /*
- Effect: Allow
Action:
Expand Down Expand Up @@ -368,12 +369,18 @@ Resources:
- LambdaExecutionRole
- Arn
Code:
S3Bucket: !Sub "${SolutionsS3BucketName}-${AWS::Region}"
S3Bucket: !Sub
- "${SolutionsS3BucketNamePrefix}-${AWS::Region}"
- SolutionsS3BucketNamePrefix:
Fn::FindInMap: [SolutionsS3BucketName, Ref: StackVersion, Prefix]
S3Key: !FindInMap
- Function
- FraudDetection
- S3Key
Runtime: python3.6
Environment:
Variables:
StackName: !Sub ${AWS::StackName}
LambdaExecutionRole:
Type: 'AWS::IAM::Role'
Properties:
Expand Down Expand Up @@ -431,7 +438,7 @@ Resources:
DeliveryStreamType: DirectPut
S3DestinationConfiguration:
BucketARN: !GetAtt
- S3Bucket2
- OutputBucket
- Arn
Prefix: !Ref KinesisFirehosePrefix
BufferingHints:
Expand Down Expand Up @@ -476,11 +483,11 @@ Resources:
- !Join
- ''
- - 'arn:aws:s3:::'
- !Ref OutputBucketName
- !Ref OutputBucket
- !Join
- ''
- - 'arn:aws:s3:::'
- !Ref OutputBucketName
- !Ref OutputBucket
- /
- !Ref KinesisFirehosePrefix
- '*'
Expand Down Expand Up @@ -630,12 +637,22 @@ Resources:
rules_to_suppress:
- id: W68
reason: Resource not associated with an AWS::ApiGateway::UsagePlan for now
SolutionAssistantStack:
Type: "AWS::CloudFormation::Stack"
Properties:
TemplateURL: ./solution-assistant/solution-assistant.yaml
Parameters:
StackName: !Ref AWS::StackName
ModelDataBucketName: !Ref ModelDataBucket
OutputBucketName: !Ref OutputBucket
LogBucketName: !Ref LogBucket
SolutionsS3BucketName: !Sub
- ${SolutionsS3BucketNamePrefix}-${AWS::Region}
- SolutionsS3BucketNamePrefix:
Fn::FindInMap: [SolutionsS3BucketName, Ref: StackVersion, Prefix]
Outputs:
JupyterInterface:
Description: "Open Jupyter IDE. This authenticates you against Jupyter."
Value: !Sub "https://console.aws.amazon.com/sagemaker/home?region=${AWS::Region}#/notebook-instances/openNotebook/FraudDetectionNotebookInstance?view=classic"
SageMakerNotebook:
Description: "Open Jupyter notebook kick off model training"
Description: "Opens the Jupyter notebook to get started with model training"
Value: !Sub "https://frauddetectionnotebookinstance.notebook.${AWS::Region}.sagemaker.aws/notebooks/source/notebooks/sagemaker_fraud_detection.ipynb"
FirehoseDeliveryStreamArn:
Description: Firehose Delivery Stream ARN
Expand Down
1 change: 1 addition & 0 deletions deployment/solution-assistant/requirements.in
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
crhelper
7 changes: 7 additions & 0 deletions deployment/solution-assistant/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile requirements.in
#
crhelper==2.0.6 # via -r requirements.in
112 changes: 112 additions & 0 deletions deployment/solution-assistant/solution-assistant.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
AWSTemplateFormatVersion: 2010-09-09
Description: Stack for Solution Helper resources.
Parameters:
StackName:
Description: |
Used as a prefix to name all stack resources.
Should be root stack's name if this is a nested stack.
Type: String
ModelDataBucketName:
Description: Amazon S3 Bucket used to store trained model and data.
Type: String
OutputBucketName:
Description: Amazon S3 Bucket used to store prediction outputs.
Type: String
LogBucketName:
Description: Amazon S3 Bucket used to S3 access logs.
Type: String
SolutionsS3BucketName:
Description: Amazon S3 Bucket containing solutions.
Type: String
Mappings:
Function:
SolutionAssistant:
S3Key: "Fraud-detection-using-machine-learning/build/solution_assistant.zip"
Resources:
SolutionAssistant:
Type: "Custom::SolutionAssistant"
Properties:
ServiceToken: !GetAtt SolutionAssistantLambda.Arn
StackName: !Ref StackName
ModelDataBucketName: !Ref ModelDataBucketName
OutputBucketName: !Ref OutputBucketName
LogBucketName: !Ref LogBucketName
SolutionsS3BucketName: !Ref SolutionsS3BucketName
SolutionAssistantLambda:
Type: AWS::Lambda::Function
Properties:
Handler: "lambda_function.handler"
FunctionName: !Sub "${StackName}-solution-assistant"
Role: !GetAtt SolutionAssistantLambdaRole.Arn
Runtime: "python3.8"
Code:
S3Bucket: !Ref SolutionsS3BucketName
S3Key: !FindInMap
- Function
- SolutionAssistant
- S3Key
Timeout : 60
SolutionAssistantLambdaRole:
Type: AWS::IAM::Role
Properties:
ManagedPolicyArns:
- arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
AssumeRolePolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Principal:
Service:
- lambda.amazonaws.com
Action:
- sts:AssumeRole
Policies:
-
PolicyName: SolutionCleanupPolicy
PolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Action:
- s3:ListBucket
Resource:
- !Sub "arn:aws:s3:::${SolutionsS3BucketName}"
- Effect: Allow
Action:
- s3:GetObject
Resource:
- !Sub "arn:aws:s3:::${SolutionsS3BucketName}/*"
- Effect: Allow
Action:
- s3:ListBucket
- s3:DeleteBucket
Resource:
- !Sub "arn:aws:s3:::${ModelDataBucketName}"
- !Sub "arn:aws:s3:::${OutputBucketName}"
- !Sub "arn:aws:s3:::${LogBucketName}"
- Effect: Allow
Action:
- s3:GetObject
- s3:PutObject
- s3:DeleteObject
- s3:AbortMultipartUpload
Resource:
- !Sub "arn:aws:s3:::${ModelDataBucketName}/*"
- !Sub "arn:aws:s3:::${OutputBucketName}/*"
- !Sub "arn:aws:s3:::${LogBucketName}/*"
- Effect: Allow
Action:
- sagemaker:DeleteModel
Resource:
- !Sub "arn:aws:sagemaker:${AWS::Region}:${AWS::AccountId}:model/*"
- Effect: Allow
Action:
- sagemaker:DeleteEndpointConfig
Resource:
- !Sub "arn:aws:sagemaker:${AWS::Region}:${AWS::AccountId}:endpoint-config/*"
- Effect: Allow
Action:
- sagemaker:DeleteEndpoint
Resource:
- !Sub "arn:aws:sagemaker:${AWS::Region}:${AWS::AccountId}:endpoint/*"
- !Sub "arn:aws:sagemaker:${AWS::Region}:${AWS::AccountId}:endpoint-config/*"
Loading

0 comments on commit 9659b7a

Please sign in to comment.