New serverless pattern: s3-lambda-dynamodb-terraform

s3-lambda-dynamodb-terraform/README.md

@@ -0,0 +1,73 @@
+# Load data from JSON files in Amazon S3 into Amazon DynamoDB using S3 Event Notification and AWS Lambda
+
+This pattern in [Terraform](https://www.terraform.io/) offers a complete solution to load data from JSON files uploaded to S3. The following resources are created:
+1. S3 Bucket with event notification on object created
+2. DynamoDB Table with on-demand billing mode
+3. Lambda function that runs python with an environment variable containing the dynamodb table name 
+
+## Requirements
+
+* [Create an AWS account](https://portal.aws.amazon.com/gp/aws/developer/registration/index.html) if you do not already have one and log in. The IAM user that you use must have sufficient permissions to make necessary AWS service calls and manage AWS resources.
+* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) installed and configured
+* [Git Installed](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
+* [Terraform](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) version 1.x (this pattern has been tested with version 1.9.8)
+
+## Deployment Instructions
+
+1. Create a new directory, navigate to that directory in a terminal and clone the GitHub repository:
+    ```bash
+    git clone https://github.com/aws-samples/serverless-patterns
+    ```
+2. Change directory to the pattern directory its source code folder:
+    ```bash
+    cd s3-lambda-dynamodb-terraform/
+    ```
+3. From the command line, use Terraform to deploy the AWS resources for the pattern as specified in the main.tf file::
+    ```
+    terraform init
+    terraform apply --auto-approve
+    ```
+4. Note the outputs from the Terraform deployment process. These contain the resource names which are used for testing.
+
+## Testing
+
+### Initiate the data load process
+1. Once deployment has completed, locate the S3 Bucket Name and DynamoDB table name in the output, for example:
+    ``` bash
+    module.s3_event.aws_s3_bucket.json_bucket: Creation complete after 2s [id=s3-lambda-dynamodb-terraform-json-store]
+
+    module.dynamodb.aws_dynamodb_table.basic-dynamodb-table: Creation complete after 7s [id=dev-test]
+    ```
+
+2. A sample JSON file is provided in the `samples` folder. You can upload it using AWS CLI:
+    ``` bash
+    aws s3 cp ./samples/test.json s3://s3-lambda-dynamodb-terraform-json-store
+    ```
+
+    > **Important**: When uploading your own JSON files, ensure they contain the following mandatory field:
+    > - `UserId`: Unique identifier for the record
+
+    Example JSON format:
+    ```json
+    {
+        "UserId": "user123",
+        "name": "John Doe",
+        "email": "[email protected]"
+    }
+    ```
+
+3. Verify the data in DynamoDB:
+    ```bash
+    aws dynamodb scan --table-name dev-test
+    ```
+
+## Documentation
+- [Amazon S3 Event Notifications](https://docs.aws.amazon.com/AmazonS3/latest/userguide/NotificationHowTo.html)
+
+## Cleanup
+
+1. Delete the stack
+    ```bash
+    terraform destroy --auto-approve
+    ```
+2. Confirm the removal and wait for the resource deletion to complete.

s3-lambda-dynamodb-terraform/example-pattern.json

@@ -0,0 +1,56 @@
+{
+    "title": "Amazon S3 to AWS Lambda to Amazon DynamoDB",
+    "description": "Upload object data from S3 to DynamoDB via Lambda.",
+    "language": "Python",
+    "level": "200",
+    "framework": "Terraform",
+    "introBox": {
+      "headline": "How it works",
+      "text": [
+        "This pattern in Terraform offers a complete solution to load data from JSON files stored on S3. The following resources are created:",
+        "- S3 Bucket with event notification on object creates",
+        "- DynamoDB Table with on-demand billing mode",
+        "- Lambda function that runs python and takes environment variables of bucket name, and dynamodb table"
+      ]
+    },
+    "gitHub": {
+      "template": {
+        "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/s3-lambda-dynamodb-terraform",
+        "templateURL": "serverless-patterns/s3-lambda-dynamodb-terraform",
+        "projectFolder": "s3-lambda-dynamodb-terraform",
+        "templateFile": "main.tf"
+      }
+    },
+    "resources": {
+      "bullets": [
+        {
+          "text": "Amazon S3 Event Notifications",
+          "link": "https://docs.aws.amazon.com/AmazonS3/latest/userguide/NotificationHowTo.html"
+        }
+      ]
+    },
+    "deploy": {
+      "text": [
+        "terraform init",
+        "terraform apply --auto-approve"
+      ]
+    },
+    "testing": {
+      "text": [
+        "See the GitHub repo for detailed testing instructions."
+      ]
+    },
+    "cleanup": {
+      "text": [
+        "Delete the stack: <code>terraform destroy --auto-approve</code>."
+      ]
+    },
+    "authors": [
+      {
+        "name": "Sarika Subramaniam",
+        "image": "link-to-your-photo.jpg",
+        "bio": "Sarika is a Solutions Architect at Amazon Web Services based in London.",
+        "linkedin": "sarika-subramaniam-9ba591118/"
+      }
+    ]
+  }

s3-lambda-dynamodb-terraform/main.tf

@@ -0,0 +1,19 @@
+provider "aws" {
+  region = var.aws_region
+  default_tags {
+    tags = {
+      Project         = var.project
+      Team            = var.team
+      CostCentre      = var.costcentre
+    }
+  }
+}
+
+module "s3_event" {
+    source = "./modules/s3_event"
+    table_name = module.dynamodb.table_name
+}
+
+module "dynamodb" {
+    source = "./modules/dynamodb"
+}

s3-lambda-dynamodb-terraform/modules/dynamodb/main.tf

@@ -0,0 +1,15 @@
+locals {
+  table_name = "${var.project}-${var.table_name}"
+}
+
+# Create the Dynamodb table
+resource "aws_dynamodb_table" "basic-dynamodb-table" {
+  name           = local.table_name
+  billing_mode   = "PAY_PER_REQUEST"
+  hash_key       = var.hash_key
+
+  attribute {
+    name = var.hash_key
+    type = "S"
+  }
+}

s3-lambda-dynamodb-terraform/modules/dynamodb/outputs.tf

@@ -0,0 +1,3 @@
+output "table_name" {
+  value = aws_dynamodb_table.basic-dynamodb-table.name
+}

s3-lambda-dynamodb-terraform/modules/dynamodb/variables.tf

@@ -0,0 +1,17 @@
+variable "table_name" {
+  description = "Name of the DynamoDB table"
+  type        = string
+  default = "test"
+}
+
+variable "hash_key" {
+  description = "Hash key for the DynamoDB table"
+  type        = string
+  default     = "UserId"
+}
+
+variable "project" {
+  description = "Project name"
+  type        = string
+  default     = "s3-lambda-dynamodb-terraform"
+}

s3-lambda-dynamodb-terraform/modules/s3_event/code/lambda_function.py

@@ -0,0 +1,34 @@
+import json
+import boto3
+import os
+
+s3 = boto3.client('s3')
+dynamodb = boto3.resource('dynamodb')
+
+# DynamoDB table name
+table_name = os.environ['DYNAMODB_TABLE_NAME']
+
+def lambda_handler(event, context):
+    try:
+        # Get the S3 bucket and key (file path) from the event
+        bucket_name = event['Records'][0]['s3']['bucket']['name']
+        key = event['Records'][0]['s3']['object']['key']
+
+        # Download the JSON file from S3
+        response = s3.get_object(Bucket=bucket_name, Key=key)
+        data = json.load(response['Body'])
+
+        # Write the data to the DynamoDB table
+        table = dynamodb.Table(table_name)
+        table.put_item(Item=data)
+
+        return {
+            'statusCode': 200,
+            'body': json.dumps('Data successfully written to DynamoDB table!')
+        }
+
+    except Exception as e:
+        return {
+            'statusCode': 500,
+            'body': json.dumps(f'Error: {str(e)}')
+        }

s3-lambda-dynamodb-terraform/modules/s3_event/code/tests/s3_put_event.json

@@ -0,0 +1,14 @@
+{
+    "Records": [
+        {
+            "s3": {
+                "bucket": {
+                    "name": "s3-lambda-dynamodb-terraform-json-store"
+                },
+                "object": {
+                    "key": "test.json"
+                }
+            }
+        }
+    ]
+}

s3-lambda-dynamodb-terraform/modules/s3_event/main.tf

@@ -0,0 +1,111 @@
+locals {
+  s3_name = "${var.project}-${var.bucket_name}"
+}
+
+data "aws_iam_policy_document" "assume_role" {
+  statement {
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["lambda.amazonaws.com"]
+    }
+
+    actions = ["sts:AssumeRole"]
+  }
+}
+
+data "archive_file" "python_lambda_package" {
+  type        = "zip"
+  source_file = "${path.module}/code/lambda_function.py"
+  output_path = "lambda_function_payload.zip"
+}
+
+resource "aws_iam_role" "iam_for_lambda" {
+  name               = "iam_for_lambda"
+  assume_role_policy = data.aws_iam_policy_document.assume_role.json
+}
+
+resource "aws_iam_policy" "write_to_ddb" {
+  name        = "write_to_ddb"
+  path        = "/"
+  description = "Write to ddb policy"
+
+  # Terraform's "jsonencode" function converts a
+  # Terraform expression result to valid JSON syntax.
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        "Sid": "Statement1",
+        "Effect": "Allow",
+        "Action": [
+          "dynamodb:UpdateTable",
+          "dynamodb:CreateTable",
+          "dynamodb:BatchWriteItem",
+          "dynamodb:PutItem",
+          "dynamodb:UpdateItem"
+        ],
+        "Resource": [
+          "*"
+        ]
+		  }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "lambda_basic_policy" {
+  role       = aws_iam_role.iam_for_lambda.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" 
+}
+
+resource "aws_iam_role_policy_attachment" "s3_read_only" {
+  role       = aws_iam_role.iam_for_lambda.name
+  policy_arn = "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess" 
+}
+
+resource "aws_iam_role_policy_attachment" "write_to_ddb" {
+  role       = aws_iam_role.iam_for_lambda.name
+  policy_arn = aws_iam_policy.write_to_ddb.arn
+}
+
+resource "aws_lambda_permission" "allow_bucket" {
+  statement_id  = "AllowExecutionFromS3Bucket"
+  action        = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.test_lambda.arn
+  principal     = "s3.amazonaws.com"
+  source_arn    = aws_s3_bucket.json_bucket.arn
+}
+
+resource "aws_lambda_function" "test_lambda" {
+  filename      = "lambda_function_payload.zip"
+  function_name = "upload_to_ddb"
+  role          = aws_iam_role.iam_for_lambda.arn
+  handler       = "lambda_function.lambda_handler"
+
+  source_code_hash = data.archive_file.python_lambda_package.output_base64sha256
+
+  runtime = "python3.12"
+
+  environment {
+    variables = {
+      DYNAMODB_TABLE_NAME = var.table_name
+    }
+  }
+}
+
+resource "aws_s3_bucket" "json_bucket" {
+  bucket = local.s3_name
+  force_destroy = true
+}
+
+resource "aws_s3_bucket_notification" "bucket_notification" {
+  bucket = aws_s3_bucket.json_bucket.id
+
+  lambda_function {
+    lambda_function_arn = aws_lambda_function.test_lambda.arn
+    events              = ["s3:ObjectCreated:*"]
+  }
+
+  depends_on = [aws_lambda_permission.allow_bucket]
+}

s3-lambda-dynamodb-terraform/modules/s3_event/variables.tf

@@ -0,0 +1,16 @@
+variable "bucket_name" {
+  description = "Name of the S3 bucket for website hosting"
+  type        = string
+  default     = "json-store"
+}
+
+variable "table_name" {
+  type        = string
+  description = "Name of the DynamoDB table"
+}
+
+variable "project" {
+  description = "Project name"
+  type        = string
+  default     = "s3-lambda-dynamodb-terraform"
+}