updating version 2.0

aws-solutions-library-samples · May 5, 2020 · 31679ef · 31679ef
1 parent 8357196
commit 31679ef
Show file tree

Hide file tree

Showing 7 changed files with 1,008 additions and 129 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.0.0] - 2020-03-10
+### Added
+- unsupervised learning model with Amazon SageMaker Random Cut Forest
+- data upsampling techniques like SMOTE for addressing imbalanced data
+- More interpretation of results
+- supervised learning model with XGBoost
+- Amazon API Gateway entrypoint
+
+### Removed
+- Supervised learning model with SageMaker linear learner
+
+
 ## [1.0.0] - 2019-05-16
 ### Added
 - initial checkin

diff --git a/deployment/build-s3-dist.sh b/deployment/build-s3-dist.sh
@@ -81,4 +81,8 @@ cp -r $source_dir/notebooks $build_dist_dir/
 # Package fraud_detection Lambda function
 echo "Packaging fraud_detection lambda"
 cd $source_dir/fraud_detection/
-zip -q -r9 $build_dist_dir/fraud_detection.zip *
+zip -q -r9 $build_dist_dir/fraud_detection.zip *
+
+# Copy installation script to $deployment_dir/dist
+echo "Copying install script to $deployment_dir/dist"
+cp $source_dir/setup/on-start.sh $build_dist_dir/notebooks
diff --git a/deployment/fraud-detection-using-machine-learning.template b/deployment/fraud-detection-using-machine-learning.template
@@ -1,6 +1,6 @@
 {
   "AWSTemplateFormatVersion": "2010-09-09",
-  "Description": "(SO0056) - fraud-detection-using-machine-learning: Solution for predicting fraud events with ML using Amazon SageMaker. Version 1",
+  "Description": "(SO0056) - fraud-detection-using-machine-learning: Solution for predicting fraud events with ML using Amazon SageMaker. Version 2",
   "Parameters": {
     "S3BucketName1": {
       "Type": "String",
@@ -56,6 +56,16 @@
         "S3Bucket": "%%BUCKET_NAME%%",
         "S3Key": "/fraud-detection-using-machine-learning/%%VERSION%%/notebooks/sagemaker_fraud_detection.ipynb"
       }
+    },
+    "Script": {
+      "Install": {
+        "S3Bucket": "%%BUCKET_NAME%%",
+        "S3Key": "/fraud-detection-using-machine-learning/%%VERSION%%/notebooks/on-start.sh"
+      },
+      "GenerateTraffic": {
+        "S3Bucket": "%%BUCKET_NAME%%",
+        "S3Key": "/fraud-detection-using-machine-learning/%%VERSION%%/notebooks/generate_endpoint_traffic.py"
+      }
     }
   },
   "Resources": {
@@ -169,7 +179,35 @@
                   "Fn::Join": ["", ["sed -i 's/fraud-detection-end-to-end-demo/", {
                     "Ref": "S3BucketName1"
                   }, "/g' sagemaker_fraud_detection.ipynb"]]
-                }
+                },
+                {
+                  "Fn::Join": ["", ["aws s3 cp s3://", {
+                    "Fn::Join": ["-", [{
+                      "Fn::FindInMap": ["Script", "GenerateTraffic", "S3Bucket"]
+                    }, {
+                      "Ref": "AWS::Region"
+                    }]]
+                  }, {
+                    "Fn::FindInMap": ["Script", "GenerateTraffic", "S3Key"]
+                  }, " ."]]
+                },
+                {
+                  "Fn::Join": ["", ["aws s3 cp s3://", {
+                    "Fn::Join": ["-", [{
+                      "Fn::FindInMap": ["Script", "Install", "S3Bucket"]
+                    }, {
+                      "Ref": "AWS::Region"
+                    }]]
+                  }, {
+                    "Fn::FindInMap": ["Script", "Install", "S3Key"]
+                  }, " ."]]
+                },
+                {
+                  "Fn::Join": ["", ["sed -i 's/fraud-detection-api-placeholder/", {
+                    "Ref": "RESTAPIGateway"
+                  }, "/g' generate_endpoint_traffic.py"]]
+                },
+                "bash ./on-start.sh"
               ]]
             }
           }
@@ -243,7 +281,9 @@
                 "sagemaker:CreateEndpointConfig",
                 "sagemaker:DescribeEndpoint",
                 "sagemaker:DescribeEndpointConfig",
-                "sagemaker:DeleteEndpoint"
+                "sagemaker:DeleteEndpoint",
+                "sagemaker:DeleteEndpointConfig",
+                "sagemaker:InvokeEndpoint"
               ],
               "Resource": [{
                 "Fn::Join": ["", ["arn:aws:sagemaker:", {
@@ -333,6 +373,35 @@
               "Resource": [{
                 "Fn::GetAtt": ["NotebookInstanceExecutionRole", "Arn"]
               }]
+            },
+            {
+              "Effect": "Allow",
+              "Action": [
+                "lambda:InvokeFunction"
+              ],
+              "Resource": [{
+                "Fn::GetAtt": ["LambdaFunction", "Arn"]
+              }]
+            },
+            {
+              "Effect": "Allow",
+              "Action": [
+                "execute-api:Invoke"
+              ],
+              "Resource": [{
+                "Fn::Join": [
+                  "",
+                  [
+                      "arn:aws:execute-api:",
+                      { "Ref": "AWS::Region" },
+                      ":",
+                      { "Ref": "AWS::AccountId" },
+                      ":",
+                      { "Ref": "RESTAPIGateway" },
+                      "/*/POST/*"
+                  ]
+                ]
+              }]
             }
           ]
         },
@@ -375,39 +444,6 @@
         "Runtime": "python3.6"
       }
     },
-    "ScheduledRule": {
-      "Type": "AWS::Events::Rule",
-      "Properties": {
-        "Description": "ScheduledRule",
-        "ScheduleExpression": "rate(1 minute)",
-        "State": "DISABLED",
-        "Targets": [{
-          "Arn": {
-            "Fn::GetAtt": [
-              "LambdaFunction",
-              "Arn"
-            ]
-          },
-          "Id": "TargetFunctionV1"
-        }]
-      }
-    },
-    "PermissionForEventsToInvokeLambda": {
-      "Type": "AWS::Lambda::Permission",
-      "Properties": {
-        "FunctionName": {
-          "Ref": "LambdaFunction"
-        },
-        "Action": "lambda:InvokeFunction",
-        "Principal": "events.amazonaws.com",
-        "SourceArn": {
-          "Fn::GetAtt": [
-            "ScheduledRule",
-            "Arn"
-          ]
-        }
-      }
-    },
     "LambdaExecutionRole": {
       "Type": "AWS::IAM::Role",
       "Properties": {
@@ -577,6 +613,164 @@
           }]
         }
       }
+    },
+    "RESTAPIGateway" : {
+      "Type" : "AWS::ApiGateway::RestApi",
+      "Properties" : {
+          "Description" : "A REST API that can be used to invoke the Lambda function that triggers predictions.",
+          "Name" : "model-invocation-api",
+          "EndpointConfiguration": {
+              "Types": ["REGIONAL"]
+            }
+        }
+    },
+    "APIGatewayCloudWatchLogGroup": {
+      "Type" : "AWS::Logs::LogGroup",
+      "Properties" : {
+        "LogGroupName" : {"Fn::Join": ["/", ["/aws/apigateway/AccessLogs", {"Ref": "RESTAPIGateway"}, "prod"]]},
+        "RetentionInDays" : 3653
+      }
+    },
+    "APIGatewayCloudWatchRole": {
+      "Type": "AWS::IAM::Role",
+      "Properties": {
+        "AssumeRolePolicyDocument": {
+          "Version": "2012-10-17",
+          "Statement": [
+            {
+              "Effect": "Allow",
+                "Principal": {
+                  "Service": [
+                    "apigateway.amazonaws.com"
+                  ]
+                },
+                "Action": "sts:AssumeRole"
+            }
+          ]
+        },
+        "Path": "/",
+          "ManagedPolicyArns": [
+            "arn:aws:iam::aws:policy/service-role/AmazonAPIGatewayPushToCloudWatchLogs"
+          ]
+      }
+    },
+    "APIGatewayAccount": {
+      "Type": "AWS::ApiGateway::Account",
+      "Properties": {
+        "CloudWatchRoleArn": {
+          "Fn::GetAtt": [
+            "APIGatewayCloudWatchRole",
+            "Arn"
+          ]
+        }
+      },
+      "DependsOn": ["RESTAPIGateway"]
+    },
+    "LambdaAPIPermission" : {
+      "Type": "AWS::Lambda::Permission",
+      "Properties": {
+        "Action": "lambda:InvokeFunction",
+        "FunctionName": {"Fn::GetAtt": ["LambdaFunction", "Arn"]},
+        "Principal": "apigateway.amazonaws.com",
+        "SourceArn": {
+            "Fn::Join": [
+                "",
+                [
+                    "arn:aws:execute-api:",
+                    { "Ref": "AWS::Region" },
+                    ":",
+                    { "Ref": "AWS::AccountId" },
+                    ":",
+                    { "Ref": "RESTAPIGateway" },
+                    "/*/POST/*"
+                ]
+            ]
+        }
+      }
+    },
+    "RESTInvocationResource" :{
+      "Type" : "AWS::ApiGateway::Resource",
+      "Properties" : {
+        "ParentId" : { "Fn::GetAtt": ["RESTAPIGateway", "RootResourceId"] },
+        "PathPart" : "invocations",
+        "RestApiId" : {"Ref" : "RESTAPIGateway"}
+      }
+    },
+    "POSTMethod": {
+      "Type": "AWS::ApiGateway::Method",
+      "Properties": {
+          "RestApiId": {
+              "Ref": "RESTAPIGateway"
+          },
+          "ResourceId": {
+              "Ref": "RESTInvocationResource"
+          },
+          "HttpMethod": "POST",
+          "AuthorizationType": "AWS_IAM",
+          "Integration": {
+            "Type": "AWS",
+            "IntegrationHttpMethod": "POST",
+            "Uri": {"Fn::Join" : ["", ["arn:aws:apigateway:", {"Ref": "AWS::Region"}, ":lambda:path/2015-03-31/functions/", {"Fn::GetAtt": ["LambdaFunction", "Arn"]}, "/invocations"]]},
+            "IntegrationResponses": [{
+            "ResponseTemplates": {
+              "application/json": ""
+              },
+              "StatusCode": 200
+              }, {
+                "SelectionPattern": "^not found.*",
+                "ResponseTemplates": {
+                  "application/json": "{}"
+                },
+                "StatusCode": 404
+            }],
+            "PassthroughBehavior": "WHEN_NO_TEMPLATES",
+            "RequestTemplates": {
+              "application/json":
+              "{\"data\": $input.json('$.data'),\"metadata\": $input.json('$.metadata'),\"model\": \"$input.params('model')\"}"
+            }
+          },
+          "MethodResponses": [{
+            "ResponseModels": {
+              "application/json": "Empty"
+            },
+            "StatusCode": 200
+            }, {
+              "ResponseModels": {
+                "application/json": "Empty"
+              },
+              "StatusCode": 404
+          }],
+          "RequestParameters": {
+            "method.request.querystring.model": false
+          }
+      }
+    },
+    "RestApiDeployment": {
+      "Type": "AWS::ApiGateway::Deployment",
+      "Properties": {
+        "StageDescription": {
+          "AccessLogSetting": {
+            "DestinationArn" : {"Fn::GetAtt": ["APIGatewayCloudWatchLogGroup", "Arn"]},
+            "Format" : {
+              "Fn::Join":  [",", ["{\"requestId\":\"$context.requestId\"",
+                                  "\"ip\": \"$context.identity.sourceIp\"",
+                                  "\"caller\":\"$context.identity.caller\"",
+                                  "\"user\":\"$context.identity.user\"",
+                                  "\"requestTime\":\"$context.requestTime\"",
+                                  "\"httpMethod\":\"$context.httpMethod\"",
+                                  "\"resourcePath\":\"$context.resourcePath\"",
+                                  "\"status\":\"$context.status\"",
+                                  "\"protocol\":\"$context.protocol\"",
+                                  "\"responseLength\":\"$context.responseLength\"}"
+                                 ]
+                           ]
+                    }
+          }
+        },
+        "RestApiId": {"Ref": "RESTAPIGateway"},
+        "StageName": "prod"
+      },
+      "DependsOn": ["POSTMethod"]
     }
   },
   "Outputs": {
@@ -602,6 +796,9 @@
           "Arn"
         ]
       }
+    },
+    "RestApiId" : {
+      "Value": {"Ref": "RESTAPIGateway"}
     }
   }
 }