GCP Terraform modular refactor

examples/simple/grant-permissions.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Script to grant required IAM permissions to fix Terraform permission errors
+# Replace PROJECT_ID and USER_EMAIL with your actual values
+
+PROJECT_ID=<PROJECT_ID>
+USER_EMAIL=<USER_EMAIL>
+
+echo "Granting required IAM permissions to $USER_EMAIL for project $PROJECT_ID..."
+
+# Grant Service Account Admin role for IAM operations
+echo "Granting Service Account Admin role..."
+gcloud projects add-iam-policy-binding $PROJECT_ID \
+  --member="user:$USER_EMAIL" \
+  --role="roles/iam.serviceAccountAdmin"
+
+# Alternative: Grant specific workload identity permission (less permissive)
+echo "Granting Workload Identity User role..."
+gcloud projects add-iam-policy-binding $PROJECT_ID \
+  --member="user:$USER_EMAIL" \
+  --role="roles/iam.workloadIdentityUser"
+
+# Grant Container Admin role for full cluster access
+echo "Granting Container Admin role for GKE cluster management..."
+gcloud projects add-iam-policy-binding $PROJECT_ID \
+  --member="user:$USER_EMAIL" \
+  --role="roles/container.admin"
+
+echo "Permissions granted! You should now be able to run terraform apply successfully."

examples/simple/main.tf

@@ -1,25 +1,3 @@
-terraform {
-  required_version = ">= 1.0"
-
-  required_providers {
-    google = {
-      source  = "hashicorp/google"
-      version = ">= 6.0"
-    }
-    kubernetes = {
-      source  = "hashicorp/kubernetes"
-      version = "~> 2.0"
-    }
-    helm = {
-      source  = "hashicorp/helm"
-      version = "~> 2.0"
-    }
-    random = {
-      source  = "hashicorp/random"
-      version = "~> 3.0"
-    }
-  }
-}
 
 provider "google" {
   project = var.project_id
@@ -30,158 +8,228 @@ provider "google" {
 data "google_client_config" "default" {}
 
 provider "kubernetes" {
-  host                   = "https://${module.materialize.gke_cluster.endpoint}"
+  host                   = "https://${module.gke.cluster_endpoint}"
   token                  = data.google_client_config.default.access_token
-  cluster_ca_certificate = base64decode(module.materialize.gke_cluster.ca_certificate)
+  cluster_ca_certificate = base64decode(module.gke.cluster_ca_certificate)
 }
 
 provider "helm" {
   kubernetes {
-    host                   = "https://${module.materialize.gke_cluster.endpoint}"
+    host                   = "https://${module.gke.cluster_endpoint}"
     token                  = data.google_client_config.default.access_token
-    cluster_ca_certificate = base64decode(module.materialize.gke_cluster.ca_certificate)
+    cluster_ca_certificate = base64decode(module.gke.cluster_ca_certificate)
   }
 }
 
-module "materialize" {
-  # Referencing the root module directory:
-  source = "../.."
 
-  # Alternatively, you can use the GitHub source URL:
-  # source = "github.com/MaterializeInc/terraform-google-materialize?ref=v0.1.0"
+locals {
+  common_labels = merge(var.labels, {
+    managed_by = "terraform"
+    module     = "materialize"
+  })
+
+  # Disk support configuration
+  disk_config = {
+    install_openebs           = var.enable_disk_support ? lookup(var.disk_support_config, "install_openebs", true) : false
+    run_disk_setup_script     = var.enable_disk_support ? lookup(var.disk_support_config, "run_disk_setup_script", true) : false
+    local_ssd_count           = lookup(var.disk_support_config, "local_ssd_count", 1)
+    create_storage_class      = var.enable_disk_support ? lookup(var.disk_support_config, "create_storage_class", true) : false
+    openebs_version           = lookup(var.disk_support_config, "openebs_version", "4.2.0")
+    openebs_namespace         = lookup(var.disk_support_config, "openebs_namespace", "openebs")
+    storage_class_name        = lookup(var.disk_support_config, "storage_class_name", "openebs-lvm-instance-store-ext4")
+    storage_class_provisioner = "local.csi.openebs.io"
+    storage_class_parameters = {
+      storage  = "lvm"
+      fsType   = "ext4"
+      volgroup = "instance-store-vg"
+    }
+  }
 
-  project_id = var.project_id
-  region     = var.region
-  prefix     = var.prefix
+  metadata_backend_url = format(
+    "postgres://%s:%s@%s:5432/%s?sslmode=disable",
+    var.database_config.username,
+    random_password.database_password.result,
+    module.database.private_ip,
+    var.database_config.db_name
+  )
 
-  network_config = {
-    subnet_cidr   = "10.0.0.0/20"
-    pods_cidr     = "10.48.0.0/14"
-    services_cidr = "10.52.0.0/20"
-  }
+  encoded_endpoint = urlencode("https://storage.googleapis.com")
+  encoded_secret   = urlencode(module.storage.hmac_secret)
 
-  database_config = {
-    tier     = "db-custom-2-4096"
-    version  = "POSTGRES_15"
-    password = random_password.pass.result
-  }
+  persist_backend_url = format(
+    "s3://%s:%s@%s/materialize?endpoint=%s&region=%s",
+    module.storage.hmac_access_id,
+    local.encoded_secret,
+    module.storage.bucket_name,
+    local.encoded_endpoint,
+    var.region
+  )
+}
 
-  labels = {
-    environment = "simple"
-    example     = "true"
-  }
+module "networking" {
+  source = "../../modules/networking"
 
-  install_materialize_operator = true
-  operator_version             = var.operator_version
-  orchestratord_version        = var.orchestratord_version
+  project_id    = var.project_id
+  region        = var.region
+  prefix        = var.prefix
+  subnet_cidr   = var.network_config.subnet_cidr
+  pods_cidr     = var.network_config.pods_cidr
+  services_cidr = var.network_config.services_cidr
+}
 
-  install_cert_manager           = var.install_cert_manager
-  use_self_signed_cluster_issuer = var.use_self_signed_cluster_issuer
+module "gke" {
+  source = "../../modules/gke"
 
-  # Once the operator is installed, you can define your Materialize instances here.
-  materialize_instances = var.materialize_instances
+  depends_on = [module.networking]
 
-  providers = {
-    google     = google
-    kubernetes = kubernetes
-    helm       = helm
-  }
+  project_id   = var.project_id
+  region       = var.region
+  prefix       = var.prefix
+  network_name = module.networking.network_name
+  subnet_name  = module.networking.subnet_name
+  namespace = var.namespace
 }
 
-variable "project_id" {
-  description = "GCP Project ID"
-  type        = string
-}
 
-variable "region" {
-  description = "GCP Region"
-  type        = string
-  default     = "us-central1"
+module "nodepool" {
+  source = "../../modules/nodepool"
+  depends_on = [module.gke]
+
+  nodepool_name = "${var.prefix}-node-pool"
+  region = var.region
+  enable_private_nodes = true
+  cluster_name = module.gke.cluster_name
+  project_id = var.project_id
+  node_count = var.gke_config.node_count
+  min_nodes = var.gke_config.min_nodes
+  max_nodes = var.gke_config.max_nodes
+  machine_type = var.gke_config.machine_type
+  disk_size_gb = var.gke_config.disk_size_gb
+  service_account_email = module.gke.service_account_email
+  labels = local.common_labels
+
+  disk_setup_image = var.disk_setup_image
+  enable_disk_setup = local.disk_config.run_disk_setup_script
+  local_ssd_count = local.disk_config.local_ssd_count
 }
 
-variable "prefix" {
-  description = "Used to prefix the names of the resources"
-  type        = string
-  default     = "mz-simple"
+module "openebs" {
+  source = "../../modules/openebs"
+  depends_on = [
+    module.gke,
+    module.nodepool
+  ]
+
+  install_openebs = local.disk_config.install_openebs
+  create_namespace = true
+  openebs_namespace = local.disk_config.openebs_namespace
+  openebs_version = local.disk_config.openebs_version
 }
 
-resource "random_password" "pass" {
+resource "random_password" "database_password" {
   length  = 20
   special = false
 }
 
-output "gke_cluster" {
-  description = "GKE cluster details"
-  value       = module.materialize.gke_cluster
-  sensitive   = true
-}
+module "database" {
+  source = "../../modules/database"
 
-output "service_accounts" {
-  description = "Service account details"
-  value       = module.materialize.service_accounts
-}
+  depends_on = [
+    module.networking,
+  ]
 
-output "connection_strings" {
-  description = "Connection strings for metadata and persistence backends"
-  value       = module.materialize.connection_strings
-  sensitive   = true
-}
+  database_name = var.database_config.db_name
+  database_user = var.database_config.username
 
-output "load_balancer_details" {
-  description = "Details of the Materialize instance load balancers."
-  value       = module.materialize.load_balancer_details
-}
+  project_id = var.project_id
+  region     = var.region
+  prefix     = var.prefix
+  network_id = module.networking.network_id
 
-variable "operator_version" {
-  description = "Version of the Materialize operator to install"
-  type        = string
-  default     = null
-}
+  tier       = var.database_config.tier
+  db_version = var.database_config.version
+  password   = random_password.database_password.result
 
-output "network" {
-  description = "Network details"
-  value       = module.materialize.network
+  labels = local.common_labels
 }
 
-variable "orchestratord_version" {
-  description = "Version of the Materialize orchestrator to install"
-  type        = string
-  default     = null
-}
+module "storage" {
+  source = "../../modules/storage"
 
-variable "materialize_instances" {
-  description = "List of Materialize instances to be created."
-  type = list(object({
-    name                    = string
-    namespace               = optional(string)
-    database_name           = string
-    create_database         = optional(bool, true)
-    create_load_balancer    = optional(bool, true)
-    internal_load_balancer  = optional(bool, true)
-    environmentd_version    = optional(string)
-    cpu_request             = optional(string, "1")
-    memory_request          = optional(string, "1Gi")
-    memory_limit            = optional(string, "1Gi")
-    in_place_rollout        = optional(bool, false)
-    request_rollout         = optional(string)
-    force_rollout           = optional(string)
-    balancer_memory_request = optional(string, "256Mi")
-    balancer_memory_limit   = optional(string, "256Mi")
-    balancer_cpu_request    = optional(string, "100m")
-    license_key             = optional(string)
-    environmentd_extra_args = optional(list(string), [])
-  }))
-  default = []
-}
+  project_id      = var.project_id
+  region          = var.region
+  prefix          = var.prefix
+  service_account = module.gke.workload_identity_sa_email
+  versioning      = var.storage_bucket_versioning
+  version_ttl     = var.storage_bucket_version_ttl
 
-variable "install_cert_manager" {
-  description = "Whether to install cert-manager."
-  type        = bool
-  default     = true
+  labels = local.common_labels
 }
 
-variable "use_self_signed_cluster_issuer" {
-  description = "Whether to install and use a self-signed ClusterIssuer for TLS. To work around limitations in Terraform, this will be treated as `false` if no materialize instances are defined."
-  type        = bool
-  default     = true
+module "certificates" {
+  source = "../../modules/certificates"
+
+  install_cert_manager           = var.install_cert_manager
+  cert_manager_install_timeout   = var.cert_manager_install_timeout
+  cert_manager_chart_version     = var.cert_manager_chart_version
+  use_self_signed_cluster_issuer = var.install_materialize_instance
+  cert_manager_namespace         = var.cert_manager_namespace
+  name_prefix                    = var.prefix
+
+  depends_on = [
+    module.gke,
+    module.nodepool,
+  ]
+}
+
+module "operator" {
+  count = var.install_materialize_operator ? 1 : 0
+  source = "../../modules/operator"
+
+  name_prefix                    = var.prefix
+  use_self_signed_cluster_issuer = var.install_materialize_instance
+  region = var.region
+
+  depends_on = [
+    module.gke,
+    module.nodepool,
+    module.database,
+    module.storage,
+    module.certificates,
+  ]
+}
+
+module "materialize_instance" {
+  count = var.install_materialize_instance ? 1 : 0
+
+  source               = "../../modules/materialize-instance"
+  instance_name        = "main"
+  instance_namespace   = "materialize-environment"
+  metadata_backend_url = local.metadata_backend_url
+  persist_backend_url  = local.persist_backend_url
+
+  depends_on = [
+    module.gke,
+    module.database,
+    module.storage,
+    module.networking,
+    module.certificates,
+    module.operator,
+    module.nodepool,
+    module.openebs,
+  ]
+}
+
+module "load_balancers" {
+  count = var.install_materialize_instance ? 1 : 0
+
+  source = "../../modules/load_balancers"
+
+  instance_name = "main"
+  namespace     = "materialize-environment"
+  resource_id   = module.materialize_instance[0].instance_resource_id
+
+  depends_on = [
+    module.materialize_instance,
+  ]
 }