Merge pull request #1 from naxa-developers/feat-terraform

Feat terraform

Readme.md

@@ -87,6 +87,27 @@ You can set it up as systemd service or cronjob in your PC if required or run ma
 
    Your Lambda function should be able to execute the script with the provided configurations.
 
+### Using Terraform
+
+1. **Download & install Terraform:**
+
+   - Install terraform [here.](https://developer.hashicorp.com/terraform/install)
+
+2. **Create AWS Credentials**
+
+   - Create an IAM programmatic user :
+
+     - Configure .aws/credentials or use AWS Environment varibles for terraform authentication. [Check Here](https://registry.terraform.io/providers/hashicorp/aws/latest/docs)
+
+3. **Environment Variables**
+You can set terraform input variables using `TF_VAR_<varible-name-from-vars.tf>` if you don't want to provide them each time.
+
+4. **Run Terraform Plan/Apply**
+   - Run `terraform init` to download required providers
+   - Run `terraform plan` plan to check for infrastructure changes.
+   - Run `terraform apply` to apply terraform configurations.
+
+> **_NOTE:_**  Check `main.tf` for resources terraform creates.
 ### Streamlit
 
 You can run streamlit app to use frontend 

main.tf

@@ -0,0 +1,158 @@
+terraform {
+  required_version = "= 1.5.5"
+
+  # Live modules pin exact provider version; generic modules let consumers pin the version.
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.67.0"
+    }
+  }
+}
+
+data "aws_iam_policy_document" "assume_role" {
+  statement {
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["lambda.amazonaws.com"]
+    }
+    actions = ["sts:AssumeRole"]
+  }
+}
+
+resource "aws_iam_role" "iam_for_lambda_tm" {
+  name               = "${var.iam_lambda_role_name}-${var.environment}"
+  assume_role_policy = data.aws_iam_policy_document.assume_role.json
+}
+
+data "archive_file" "lambda" {
+  type        = "zip"
+  source_dir = "${path.module}"
+  output_path = "${path.module}/files/tm-extractor-${var.account_name}-${var.environment}.zip"
+  excludes = [".terraform", "files", "tm_extractor_python3_layer.zip"]
+}
+
+resource "aws_lambda_layer_version" "lambda_layer" {
+  filename   = "tm_extractor_python3_layer.zip"
+  layer_name = "lambda_raw_data_cron_layer-${var.account_name}-${var.environment}"
+
+  compatible_runtimes = ["python3.8"]
+
+}
+resource "aws_cloudwatch_log_group" "lambda_raw_data_cron" {
+  name = "/aws/lambda/${aws_lambda_function.lambda_raw_data.function_name}"
+  retention_in_days = var.cw_retention_in_days
+  lifecycle {
+    prevent_destroy = false
+  }
+  tags = {
+  Application = "lambda"
+  }
+}
+
+resource "aws_lambda_function" "lambda_raw_data" {
+
+  filename = "${var.zip_output_dir}/tm-extractor-${var.account_name}-${var.environment}.zip"
+  function_name = "tm_extractor-${var.account_name}-${var.environment}"
+  role          = aws_iam_role.iam_for_lambda_tm.arn
+  handler       = "tm_extractor.lambda_handler"
+  memory_size   = var.lambda_memory_size
+  timeout = var.lambda_timeout # To be Increased if active projects are more.
+  layers = [ aws_lambda_layer_version.lambda_layer.id ]
+
+  source_code_hash = data.archive_file.lambda.output_base64sha256
+
+  runtime = "python3.9"
+  # To be accessed from Environmnet varible TG or TF Environment Variables. See: https://terragrunt.gruntwork.io/docs/features/inputs/
+  depends_on = [
+    aws_lambda_layer_version.lambda_layer,
+    aws_iam_role.iam_for_lambda_tm,
+    aws_iam_role_policy_attachment.lambda_logs,
+  ]
+  environment {
+    variables = {
+      TM_API_BASE_URL = "${var.active_projects_api_base_url}",
+      RAWDATA_API_AUTH_TOKEN = "${var.rawdata_api_auth_token}"
+      RAW_DATA_API = "${var.raw_data_api}"
+      CONFIG_JSON = "${var.config_json}"
+    }
+  }
+
+  tracing_config {
+    mode = "PassThrough"
+  }
+  lifecycle {
+    ignore_changes = [layers]  # Ignore changes to layers for now
+  }
+}
+
+resource "aws_iam_policy" "lambda_logging" {
+  name        = "lambda_raw_data_logging-${var.account_name}-${var.environment}"
+  path        = "/"
+  description = "IAM policy for logging from this lambda function"
+
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": [
+        "logs:CreateLogGroup",
+        "logs:CreateLogStream",
+        "logs:PutLogEvents"
+      ],
+      "Resource": "arn:aws:logs:*:*:*",
+      "Effect": "Allow"
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy_attachment" "lambda_logs" {
+  role       = aws_iam_role.iam_for_lambda_tm.name
+  policy_arn = aws_iam_policy.lambda_logging.arn
+}
+
+resource "aws_cloudwatch_event_rule" "cron_raw_data_lambda_schedule" {
+  name        = "cron_raw_data_lambda_schedule-${var.account_name}-${var.environment}"
+  description = "Schedule rule to trigger Lambda"
+  schedule_expression = var.lambda_cron_expression  # Runs day 12 am.
+}
+
+resource "aws_lambda_permission" "eventbridge_invoke_permission" {
+  statement_id  = "AllowExecutionFromEventBridge"
+  action        = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.lambda_raw_data.function_name
+  principal     = "events.amazonaws.com"
+  source_arn    = aws_cloudwatch_event_rule.cron_raw_data_lambda_schedule.arn
+}
+
+resource "aws_cloudwatch_event_target" "cron_raw_data_lambda_target" {
+  rule      = aws_cloudwatch_event_rule.cron_raw_data_lambda_schedule.name
+  arn       = aws_lambda_function.lambda_raw_data.arn
+}
+
+resource "aws_cloudwatch_metric_alarm" "lambda_error_alarm" {
+  alarm_name          = "lambda-raw-data-error-alarm-${var.account_name}-${var.environment}"
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+  evaluation_periods  = 5
+  threshold           = 2  # Number of errors that trigger the alarm.
+  period              = 14400  # 1 Day
+  namespace           = "AWS/Lambda"
+  metric_name         = "Errors"
+  statistic           = "Maximum"
+  dimensions = {
+    FunctionName = aws_lambda_function.lambda_raw_data.function_name
+  }
+
+  alarm_description = "Alarm triggered when Lambda function has 3 or more errors in 5 minutes."
+
+  actions_enabled = false  # Disable actions, meaning no notification actions will be taken
+
+  treat_missing_data = "breaching"
+}
+
+

vars.tf

@@ -0,0 +1,80 @@
+# To be used by main.tf & managed through terragrunt.
+# Check 
+
+variable "aws_region" {
+  description = "The AWS region to deploy to (e.g. us-east-1)"
+  type        = string
+  default = "ap-south-1"
+}
+
+variable "account_name" {
+  type = string
+  default = "hotosm"
+  description = "AWS Account name, acts as suffix to variour resources"
+}
+
+variable "iam_lambda_role_name" {
+  description = "The name of the role lambda functions runs as"
+  type        = string
+}
+
+variable "cw_retention_in_days" {
+  description = "Defines number of days cloudwatch logs are retained. Example: 1 or 10 Years"
+  type        = string
+  default = "14"
+}
+
+variable "environment" {
+  description = "The Environment lambda functions are deploymed on. Changes the prefix and names of resources. eg: dev, stag, prod"
+  type        = string
+  default = "dev"
+}
+
+variable "lambda_memory_size" {
+  description = "Defines the memory size the lambda functions runs on. Default 128 MB"
+  type        = string
+  default = "128"
+}
+
+variable "lambda_timeout" {
+  description = "Defines the timeout for the lambda function. Increase if lambda function times out. Default 20 sec."
+  type        = string
+  default = "20"
+}
+
+variable "lambda_cron_expression" {
+  description = "Defines the Cron expression for the lambda function to run. Defaults to everyday at 00:00 am"
+  type        = string
+  default = "cron(0 0 * * ? *)"
+}
+# To be Exported from environment as TG_active_projects_api_base_url, from circleci or gh actions.
+variable "active_projects_api_base_url" {
+   description = "Link to your tasking manage project"
+   type = string
+   default = "https://tasking-manager-staging-api.hotosm.org/api/v2"
+}
+
+# To be Exported from environment as TG_rawdata_api_auth_token, from circleci or gh actions.
+variable "rawdata_api_auth_token" {
+   type = string
+   description = "API Token for rawdata api, get your token from https://api-prod.raw-data.hotosm.org/v1"
+}
+
+# To be Exported from environment as TG_raw_data_api, from circleci or gh actions.
+variable "raw_data_api" {
+   type = string
+   description = "Link to raw data api"
+   default = "https://api-prod.raw-data.hotosm.org/v1"
+}
+
+variable "zip_output_dir" {
+   description = "This is a path to temporary folder directory due to TG limitations, check: https://github.com/gruntwork-io/terragrunt/issues/716"
+   type = string
+   default = "files"
+}
+
+variable "config_json" {
+   description = "Path to config.json, if the file exits in another directory."
+   type = string
+   default = "config.json"
+}