From 987c03e093357cdbfcf90d422834cf88055d4f6c Mon Sep 17 00:00:00 2001 From: Soji Adeshina Date: Mon, 13 Jul 2020 14:36:57 -0700 Subject: [PATCH] update README to new structure --- README.md | 9 +++- ...raud-detection-using-machine-learning.yaml | 12 +---- source/notebooks/generate_endpoint_traffic.py | 48 ------------------- .../notebooks/sagemaker_fraud_detection.ipynb | 3 +- source/notebooks/src/package/config.py | 1 - 5 files changed, 10 insertions(+), 63 deletions(-) delete mode 100644 source/notebooks/generate_endpoint_traffic.py diff --git a/README.md b/README.md index b62c636..b56e5e8 100644 --- a/README.md +++ b/README.md @@ -44,10 +44,15 @@ In order to encapsulate the project as a stand-alone microservice, Amazon API Ga * `deployment/` * `fraud-detection-using-machine-learning.yaml`: Creates AWS CloudFormation Stack for solution * `source/` - * `fraud-detection/` + * `lambda` + * `model-invocation/` * `index.py`: Lambda function script for invoking SageMaker endpoints for inference * `notebooks/` - * `generate_endpoint_traffic.py`: Custom script to show how to send transaction traffic to REST API for inference + * `src` + * `package` + * `config.py`: Read in the environment variables set by cloudformation stack creation + * `generate_endpoint_traffic.py`: Custom script to show how to send transaction traffic to REST API for inference + * `util.py`: Helper function and utilities * `sagemaker_fraud_detection.ipynb`: Orchestrates the solution. Trains the models and deploys the trained model * `setup/` * `on-start.sh`: Bash script to setup sagemaker notebook environment with necessary dependencies diff --git a/deployment/fraud-detection-using-machine-learning.yaml b/deployment/fraud-detection-using-machine-learning.yaml index 324a68b..44e81c2 100644 --- a/deployment/fraud-detection-using-machine-learning.yaml +++ b/deployment/fraud-detection-using-machine-learning.yaml @@ -45,17 +45,7 @@ Metadata: Mappings: Function: FraudDetection: - S3Key: fraud-detection-using-machine-learning/build/model_invocation.zip - Notebook: - FraudDetection: - S3Key: >- - /fraud-detection-using-machine-learning/notebooks/sagemaker_fraud_detection.ipynb - Script: - GenerateTraffic: - S3Key: >- - /fraud-detection-using-machine-learning/notebooks/generate_endpoint_traffic.py - KibanaDashboard: - S3Key: /fraud-detection-using-machine-learning/notebooks/dashboard.json + S3Key: Fraud-detection-using-machine-learning/build/model_invocation.zip Resources: ModelDataBucket: Type: 'AWS::S3::Bucket' diff --git a/source/notebooks/generate_endpoint_traffic.py b/source/notebooks/generate_endpoint_traffic.py deleted file mode 100644 index 5c6a782..0000000 --- a/source/notebooks/generate_endpoint_traffic.py +++ /dev/null @@ -1,48 +0,0 @@ -import time -import boto3 -import json -import re -import datetime -import random - -import requests -from aws_requests_auth.boto_utils import BotoAWSRequestsAuth -import numpy as np -from scipy.stats import poisson - -lambda_client = boto3.client('lambda') -lambda_function = 'fraud-detection-event-processor' -region = boto3.Session().region_name - -def generate_metadata(): - millisecond_regex = r'\.\d+' - timestamp = re.sub(millisecond_regex, '', str(datetime.datetime.now())) - source = random.choice(['Mobile', 'Web', 'Store']) - result = [timestamp, 'random_id', source] - - return result - -def get_data_payload(test_array): - return {'data':','.join(map(str, test_array)), - 'metadata': generate_metadata()} - -def generate_traffic(X_test): - while True: - np.random.shuffle(X_test) - for example in X_test: - data_payload = get_data_payload(example) - invoke_endpoint(data_payload) - # We invoke the function according to a shifted Poisson distribution - # to simulate data arriving at random intervals - time.sleep(poisson.rvs(1, size=1)[0] + np.random.rand()) - -def invoke_endpoint(payload): - # We get credentials from the IAM role of the notebook instance, then use them to create a signed request to the API Gateway - auth = BotoAWSRequestsAuth(aws_host="fraud-detection-api-placeholder.execute-api.{}.amazonaws.com".format(region), - aws_region=region, - aws_service='execute-api') - - invoke_url = "https://fraud-detection-api-placeholder.execute-api.{}.amazonaws.com/prod/invocations".format(region) - - # We don't use the response here, as the Lambda function will log any calls to it. - response = requests.post(invoke_url, json=payload, auth=auth) diff --git a/source/notebooks/sagemaker_fraud_detection.ipynb b/source/notebooks/sagemaker_fraud_detection.ipynb index 99792a9..ff5ba95 100644 --- a/source/notebooks/sagemaker_fraud_detection.ipynb +++ b/source/notebooks/sagemaker_fraud_detection.ipynb @@ -157,9 +157,10 @@ "import os\n", "import sagemaker\n", "from sagemaker import get_execution_role\n", + "from package import config\n", "\n", "session = sagemaker.Session()\n", - "bucket = \"fraud-detection-end-to-end-demo\"\n", + "bucket = config.MODEL_DATA_S3_BUCKET\n", "prefix = 'fraud-classifier'" ] }, diff --git a/source/notebooks/src/package/config.py b/source/notebooks/src/package/config.py index ac189b1..131aa7d 100644 --- a/source/notebooks/src/package/config.py +++ b/source/notebooks/src/package/config.py @@ -14,7 +14,6 @@ AWS_ACCOUNT_ID = os.environ['AWS_ACCOUNT_ID'] AWS_REGION = os.environ['AWS_REGION'] SAGEMAKER_IAM_ROLE = os.environ['SAGEMAKER_IAM_ROLE'] -STACK_NAME = os.environ['STACK_NAME'] SOLUTIONS_S3_BUCKET = os.environ['SOLUTIONS_S3_BUCKET'] MODEL_DATA_S3_BUCKET = os.environ['MODEL_DATA_S3_BUCKET']