diff --git a/infrastructure/example/simple-node-pack-extract/main.tf b/infrastructure/example/simple-node-pack-extract/main.tf new file mode 100644 index 0000000..783d0a8 --- /dev/null +++ b/infrastructure/example/simple-node-pack-extract/main.tf @@ -0,0 +1,56 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "6.14.1" + } + } +} + +variable "prefix" { + type = string +} + +variable "region" { + type = string + default = "us-central1" +} + +provider "google" { + region = var.region +} + +resource "google_storage_bucket" "bucket" { + name = "${var.prefix}-comfy-registry-bucket" + location = var.region +} + +resource "google_service_account" "service_account" { + account_id = "${var.prefix}-comfy-registry-sa" +} + +module "node_pack_extract_trigger" { + depends_on = [google_service_account.service_account, google_storage_bucket.bucket] + source = "../../module/node-pack-extract-trigger" + providers = { + google = google + } + region = var.region + bucket_name = google_storage_bucket.bucket.name + cloud_build_service_account = google_service_account.service_account.email + topic_name = "${var.prefix}-comfy-registry-event" + trigger_name = "${var.prefix}-comfy-registry-event" +} + +output "trigger_id" { + value = module.node_pack_extract_trigger.trigger_id +} +output "topic_id" { + value = module.node_pack_extract_trigger.topic_id +} +output "bucket_notification_id" { + value = module.node_pack_extract_trigger.bucket_notification_id +} +output "bucket_name" { + value = google_storage_bucket.bucket.name +} diff --git a/infrastructure/module/node-pack-extract-trigger/README.md b/infrastructure/module/node-pack-extract-trigger/README.md new file mode 100644 index 0000000..bac6dee --- /dev/null +++ b/infrastructure/module/node-pack-extract-trigger/README.md @@ -0,0 +1,10 @@ +# Trigger for node-pack-extract + +Terraform modules to setup trigger for cloud build that will run [node-pack-extract](../../../node-pack-extract/) + +## Requirements + +- Google Cloud Account +- Existing Google Cloud Storage public bucket where the Registry backend store the comfy node packs. +- Existing Service Account that is whitelisted in [service_account_auth](../../../server/middleware/authentication/service_account_auth.go#65) middleware and with `Service Account Token Creator` Role. +- [Connected repositories](https://cloud.google.com/build/docs/repositories) contains the [node-pack-extract](../../../node-pack-extract/) folder diff --git a/infrastructure/module/node-pack-extract-trigger/main.tf b/infrastructure/module/node-pack-extract-trigger/main.tf new file mode 100644 index 0000000..d19f07a --- /dev/null +++ b/infrastructure/module/node-pack-extract-trigger/main.tf @@ -0,0 +1,67 @@ +# get the existing GCS bucket +data "google_storage_bucket" "bucket" { + name = var.bucket_name +} + +# create a Pub/Sub topic +resource "google_pubsub_topic" "topic" { + name = var.topic_name +} + +# get the default GCS service account +data "google_storage_project_service_account" "gcs_account" { +} + +# Grant the GCS service account permission to publish to the Pub/Sub topic +resource "google_pubsub_topic_iam_binding" "gcs_publisher" { + topic = google_pubsub_topic.topic.name + role = "roles/pubsub.publisher" + members = ["serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"] +} + +# enable GCS Bucket Notification to Pub/Sub +resource "google_storage_notification" "notification" { + bucket = data.google_storage_bucket.bucket.name + topic = google_pubsub_topic.topic.id + payload_format = "JSON_API_V1" + depends_on = [google_pubsub_topic_iam_binding.gcs_publisher] + event_types = [ + "OBJECT_FINALIZE", # Triggered when an object is successfully created or overwritten + ] +} + + +# Get the existing cloudbuild service account +data "google_service_account" "cloudbuild_service_account" { + account_id = var.cloud_build_service_account +} + +# Create the cloud build trigger +resource "google_cloudbuild_trigger" "trigger" { + name = var.trigger_name + location = var.region + service_account = data.google_service_account.cloudbuild_service_account.id + + pubsub_config { + topic = google_pubsub_topic.topic.id + } + + source_to_build { + uri = var.git_repo_uri + ref = "refs/heads/${var.git_repo_branch}" + repo_type = "GITHUB" + } + + git_file_source { + uri = var.git_repo_uri + revision = "refs/heads/${var.git_repo_branch}" + repo_type = "GITHUB" + path = "node-pack-extract/cloudbuild.yaml" + } + + substitutions = { + _CUSTOM_NODE_NAME = "custom-node" + _CUSTOM_NODE_URL = "https://storage.googleapis.com/$(body.message.data.bucket)/$(body.message.data.name)" + _REGISTRY_BACKEND_URL = var.registry_backend_url + } +} diff --git a/infrastructure/module/node-pack-extract-trigger/output.tf b/infrastructure/module/node-pack-extract-trigger/output.tf new file mode 100644 index 0000000..32cbceb --- /dev/null +++ b/infrastructure/module/node-pack-extract-trigger/output.tf @@ -0,0 +1,11 @@ +output "topic_id" { + value = google_pubsub_topic.topic.id +} + +output "bucket_notification_id" { + value = google_storage_notification.notification.id +} + +output "trigger_id" { + value = google_cloudbuild_trigger.trigger.id +} diff --git a/infrastructure/module/node-pack-extract-trigger/variable.tf b/infrastructure/module/node-pack-extract-trigger/variable.tf new file mode 100644 index 0000000..7cb5f13 --- /dev/null +++ b/infrastructure/module/node-pack-extract-trigger/variable.tf @@ -0,0 +1,49 @@ +# REQUIRED VARIABLE +variable "bucket_name" { + type = string + description = "Existing public bucket that store the comfy node-packs." +} + +variable "cloud_build_service_account" { + type = string + description = "Existing service account used to run the cloud build and used to access registry backend, e.g. cloud-build@my-project.iam.gserviceaccount.com. Note that this service account needs to have 'Service Account Token Creator' role." +} + +# OPTIONAL VARIABLE +variable "region" { + type = string + description = "Google Cloud region" + default = "us-central1" +} + +variable "topic_name" { + type = string + description = "Google Cloudpub/sub topic to be created" + default = "comfy-registry-event" +} + +variable "trigger_name" { + type = string + description = "Cloud build trigger name" + default = "comfy-registry-nodepack" + +} + +variable "git_repo_uri" { + type = string + description = "Connected git repo containing the cloud build pipeline. See https://cloud.google.com/build/docs/repositories" + default = "https://github.com/Comfy-Org/registry-backend" +} + +variable "git_repo_branch" { + type = string + description = "Git repo branch." + default = "master" +} + +variable "registry_backend_url" { + type = string + description = "The base url where registry backend can be reached" + default = "https://api.comfy.org" +} + diff --git a/infrastructure/module/node-pack-extract-trigger/version.tf b/infrastructure/module/node-pack-extract-trigger/version.tf new file mode 100644 index 0000000..7818333 --- /dev/null +++ b/infrastructure/module/node-pack-extract-trigger/version.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "6.14.1" + } + } +} diff --git a/infrastructure/prod/main.tf b/infrastructure/prod/main.tf index 075b9e0..8a7136f 100644 --- a/infrastructure/prod/main.tf +++ b/infrastructure/prod/main.tf @@ -13,7 +13,7 @@ provider "google" { } module "node_pack_extract_trigger" { - source = "../../node-pack-extract/trigger" + source = "../module/node-pack-extract-trigger" providers = { google = google } diff --git a/infrastructure/staging/main.tf b/infrastructure/staging/main.tf index 81af699..ca71d9e 100644 --- a/infrastructure/staging/main.tf +++ b/infrastructure/staging/main.tf @@ -13,12 +13,12 @@ provider "google" { } module "node_pack_extract_trigger" { - source = "../../node-pack-extract/trigger" + source = "../module/node-pack-extract-trigger" providers = { google = google } region = var.region bucket_name = "comfy-registry" cloud_build_service_account = "cloud-scheduler@dreamboothy.iam.gserviceaccount.com" - topic_name = "comfy-registry-event-stage" + topic_name = "comfy-registry-event-staging" } diff --git a/node-pack-extract/test/trigger_test.go b/node-pack-extract/test/trigger_test.go index b0ca3d6..696ad08 100644 --- a/node-pack-extract/test/trigger_test.go +++ b/node-pack-extract/test/trigger_test.go @@ -16,7 +16,7 @@ import ( ) func TestApply(t *testing.T) { - terraformDir := test_structure.CopyTerraformFolderToTemp(t, "../", "test/testdata") + terraformDir := test_structure.CopyTerraformFolderToTemp(t, "../../", "infrastructure/example/simple-node-pack-extractcd") terraformOptions := &terraform.Options{ TerraformDir: terraformDir, Vars: map[string]interface{}{