Azure vnet flow logs support in tf module

README.md

@@ -12,7 +12,7 @@
 #### Single VPC, Single Region
 * [single-vpc](https://github.com/kentik/config-snippets-cloud/tree/master/cloud_AWS/terraform/module/examples/single-vpc)
 #### All VPC, Single Region
-* [all-vpc-from-region](https://github.com/kentik/config-snippets-cloud/tree/master/cloud_AWS/terraform/module/examples/all-vpc-from-region) 
+* [all-vpc-from-region](https://github.com/kentik/config-snippets-cloud/tree/master/cloud_AWS/terraform/module/examples/all-vpc-from-region)
 #### Deploy Sock Shop as an example micro-service architecture
 * [sock-shop-eks](https://github.com/kentik/config-snippets-cloud/tree/master/cloud_AWS/terraform/module/examples/sock-shop-eks)
 
@@ -28,8 +28,6 @@
 # Stage 2 - Automate GCP
 ## Terraform
 * [Terraform](https://github.com/kentik/config-snippets-cloud/tree/master/cloud_GCP/terraform)
-### Demo
-* [Terraform Demo](https://github.com/kentik/config-snippets-cloud/tree/master/cloud_GCP/terraform/module/demo) (TODO)
 ### Examples
 #### Subnet-list, Single region
 * [subnet-list](https://github.com/kentik/config-snippets-cloud/tree/master/cloud_GCP/terraform/module/examples/subnet-list)
@@ -38,14 +36,10 @@
 
 ## Ansible
 * [Ansible](https://github.com/kentik/config-snippets-cloud/tree/master/cloud_GCP/terraform)
-### Demo
-* [Ansible Demo](https://github.com/kentik/config-snippets-cloud/tree/master/cloud_GCP/terraform/module/demo)(TODO)
 
 # Stage 3 - Automate Azure
 ## Terraform
 * [Tearraform](https://github.com/kentik/config-snippets-cloud/tree/master/cloud_Azure/terraform)
-### Demo
-* [Terraform Demo](https://github.com/kentik/config-snippets-cloud/tree/master/cloud_Azure/terraform/module/demo) (TODO)
 ### Examples
 #### Subnet-list, Single region
 * [all_nsg](https://github.com/kentik/config-snippets-cloud/tree/master/cloud_Azure/terraform/module/examples/all_nsg)
@@ -56,9 +50,6 @@
 #### All NSG from resource group
 * [all_nsg](cloud_Azure/ansible/examples/all_nsg)
 
-# Stage 4 - Automate IBM Cloud
-## Timing TBD
-
 # General needs for automation
 ## Identity and Access Management
 ## Creation of Storage location

cloud_Azure/terraform/module/cloudexport.tf

@@ -14,9 +14,9 @@ provider "kentik-cloudexport" {
 
 # Creates one Kentik CloudExport for every requested Resource Group
 resource "kentik-cloudexport_item" "azure_export" {
-  count = length(var.resource_group_names)
+  for_each = { for rg in var.resource_group_names : rg => rg }
 
-  name           = "${var.name}-${var.resource_group_names[count.index]}-${var.subscription_id}" # resource group name + subscription id make the name unique
+  name           = "${var.name}-${each.key}-${var.subscription_id}" # resource group name + subscription id make the name unique
   type           = "CLOUD_EXPORT_TYPE_KENTIK_MANAGED"
   enabled        = var.enabled
   description    = var.description
@@ -25,8 +25,8 @@ resource "kentik-cloudexport_item" "azure_export" {
   azure {
     subscription_id            = var.subscription_id
     location                   = var.location
-    resource_group             = var.resource_group_names[count.index]
-    storage_account            = azurerm_storage_account.logs_storage_account[count.index].name # storage accounts are mapped to resource groups 1:1
+    resource_group             = each.key
+    storage_account            = azurerm_storage_account.logs_storage_account[each.key].name # storage accounts are mapped to resource groups 1:1
     security_principal_enabled = true
   }
-}
+}

cloud_Azure/terraform/module/network_watcher.tf

@@ -5,55 +5,39 @@ data "azurerm_network_watcher" "network_watcher" {
   resource_group_name = "NetworkWatcherRG"
 }
 
-# Runs python script to gather network security groups from each requested resource group
-# This is required because no Terraform provider exposes such functionality
-# Resulting "data.external.nsg_data_source.results" is a map of string -> string, eg.
-# {
-#   "ResourceGroupName1" -> "NetworkSercurityGroupId1,NetworkSecurityGroupId2",
-#   "ResourceGroupName2" -> "NetworkSercurityGroupId3,NetworkSecurityGroupId4"
-# }
-data "external" "nsg_data_source" {
-  program = ["python3", "${path.module}/get_nsg.py"]
-  query = {
-    resource_group_names = join(",", var.resource_group_names)
-  }
+# Fetch all VNets for each resource group
+data "azurerm_resources" "vnet" {
+  for_each            = toset(var.resource_group_names)
+  type                = "Microsoft.Network/virtualNetworks"
+  resource_group_name = each.key
 }
 
-# Convert map of string -> string:
-# {
-#   "ResourceGroupName1" -> "NetworkSercurityGroupId1,NetworkSecurityGroupId2",
-#   "ResourceGroupName2" -> "NetworkSercurityGroupId3,NetworkSecurityGroupId4"
-# }
-# to list of objects:
-# [
-#   {rg = "ResourceGroupName1", nsg = "NetworkSercurityGroupId1"},
-#   {rg = "ResourceGroupName1", nsg = "NetworkSercurityGroupId2"},
-#   {rg = "ResourceGroupName2", nsg = "NetworkSercurityGroupId3"},
-#   {rg = "ResourceGroupName2", nsg = "NetworkSercurityGroupId4"}
-# ]
+# Map resource group names to their corresponding VNets
+# Flatten map to list of objects
 locals {
-  flat_nsgs = flatten([
-    for rg, nsg_list in data.external.nsg_data_source.result : [
-      for nsg in split(",", nsg_list) : {
-        rg  = rg  # Resource Group name
-        nsg = nsg # Network Security Group ID
+  flat_vnets = flatten([
+    for rg in var.resource_group_names : [
+      for vnet in data.azurerm_resources.vnet[rg].resources : {
+        rg   = rg
+        vnet = vnet.name
+        id   = vnet.id
       }
-    ] if length(nsg_list) > 0 # filter out Resource Groups that have no Network Security Groups
+    ]
   ])
 }
 
-# Turns on flow logs for all network security groups in requested resource groups
+# Turns on vnet flow logs for all vnets in requested resource groups
 resource "azurerm_network_watcher_flow_log" "kentik_network_flow_log" {
-  count = length(local.flat_nsgs)
+  for_each = { for vnet in local.flat_vnets : vnet.name => vnet }
 
-  name = "${var.name}_flow_log_${count.index}"
+  name                 = format("${var.name}-flowLogs-%s", each.key)
   network_watcher_name = data.azurerm_network_watcher.network_watcher.name
-  resource_group_name  = data.azurerm_network_watcher.network_watcher.resource_group_name
+  resource_group_name  = each.value.rg
 
-  network_security_group_id = local.flat_nsgs[count.index].nsg
-  storage_account_id        = azurerm_storage_account.logs_storage_account[index(var.resource_group_names, local.flat_nsgs[count.index].rg)].id
-  enabled                   = true
-  version                   = 2
+  target_resource_id = each.value.id
+  storage_account_id = azurerm_storage_account.logs_storage_account[each.value.rg].id
+  enabled            = true
+  version            = 2
   retention_policy {
     enabled = true
     days    = 7

cloud_Azure/terraform/module/output.tf

@@ -1,8 +1,3 @@
-output "network_security_groups" {
-  value       = [for v in local.flat_nsgs : v.nsg]
-  description = "Id's of the Network Security Groups which flow logs will be collected"
-}
-
 output "subscription_id" {
   value       = var.subscription_id
   description = "Azure subscription ID"
@@ -14,11 +9,11 @@ output "resource_group_names" {
 }
 
 output "storage_accounts" {
-  value       = azurerm_storage_account.logs_storage_account[*].name
+  value       = [for sa in azurerm_storage_account.logs_storage_account : sa.name]
   description = "Storage Account names where flow logs will be collected"
 }
 
 output "principal_id" {
   value       = local.kentik_nsg_flow_exporter_id
   description = "Service Principal ID created for Kentik NSG Flow Exporter application"
-}
+}

cloud_Azure/terraform/module/roles.tf

@@ -1,17 +1,17 @@
 # Provide service principal Contributor role to each storage account
 resource "azurerm_role_assignment" "kentic_role_contributor" {
-  count = length(azurerm_storage_account.logs_storage_account)
+  for_each = azurerm_storage_account.logs_storage_account
 
-  scope                = azurerm_storage_account.logs_storage_account[count.index].id
+  scope                = each.value.id
   role_definition_name = "Contributor"
   principal_id         = local.kentik_nsg_flow_exporter_id
 }
 
 # Provide service principal Reader role to each Resource Group
 resource "azurerm_role_assignment" "kentic_role_reader" {
-  count = length(var.resource_group_names)
+  for_each = toset(var.resource_group_names)
 
-  scope                = "/subscriptions/${var.subscription_id}/resourceGroups/${var.resource_group_names[count.index]}"
+  scope                = "/subscriptions/${var.subscription_id}/resourceGroups/${each.value}"
   role_definition_name = "Reader"
   principal_id         = local.kentik_nsg_flow_exporter_id
 }

cloud_Azure/terraform/module/storage_account.tf

@@ -7,19 +7,29 @@ locals {
   generated_storage_account_names = [for name in local._alphanum_lowercase_names : substr(name, 0, 24)]
 }
 
+locals {
+  # Create a map of resource group names to storage account names
+  resource_group_to_storage_account = {
+    for rg in var.resource_group_names : rg => (
+      length(var.storage_account_names) == length(var.resource_group_names) ?
+      var.storage_account_names[index(var.resource_group_names, rg)] :
+      local.generated_storage_account_names[index(var.resource_group_names, rg)]
+    )
+  }
+}
+
 # Creates one storage account per resource group to store flow logs
-# StorageAccounts are mapped 1:1 to resource_group_names and this fact is used to get storage account id for given resource group name 
+# StorageAccounts are mapped 1:1 to resource_group_names and this fact is used to get storage account id for given resource group name
 resource "azurerm_storage_account" "logs_storage_account" {
-  count = length(var.resource_group_names)
+  for_each = local.resource_group_to_storage_account
 
-  # use either custom name if one is provided, or generated one
-  name                     = length(var.storage_account_names) == length(var.resource_group_names) ? var.storage_account_names[count.index] : local.generated_storage_account_names[count.index]
-  resource_group_name      = var.resource_group_names[count.index]
+  name                     = each.value
+  resource_group_name      = each.key
   location                 = var.location
   account_tier             = "Standard"
   account_replication_type = "GRS"
 
   tags = {
     app = var.resource_tag
   }
-}
+}