add node-pack-extract

.gitignore

@@ -10,4 +10,8 @@ tmp/
 # Lazy way to make atlas usable on windows
 atlas.exe
 
-.env
+
+# terraform
+**/.terraform/*
+terraform.tfstate
+terraform.tfstate.*

infrastructure/environment/prod/README.md

@@ -0,0 +1,23 @@
+# Trigger for node-pack-extract production
+
+Terraform modules to setup trigger for cloud build that will run [node-pack-extract](../../../node-pack-extract/)
+
+## Requirements
+
+- Google Cloud Account
+
+## Configuration
+
+This use the following configuration value:
+
+- bucket_name: "comfy-registry "
+- service account: "<[email protected]>"
+- topic_name: "comfy-registry-event"
+
+## Apply
+
+```bash
+terraform apply 
+    -var project_id=dreamboothy-dev
+    -var region=us-central1
+```

infrastructure/environment/prod/main.tf

@@ -0,0 +1,24 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "6.14.1"
+    }
+  }
+}
+
+provider "google" {
+  project = var.project_id
+  region  = var.region
+}
+
+module "node_pack_extract_trigger" {
+  source = "../../modules/node-pack-extract-trigger"
+  providers = {
+    google = google
+  }
+  region                      = var.region
+  bucket_name                 = "comfy-registry "
+  cloud_build_service_account = "[email protected]"
+  topic_name                  = "comfy-registry-event"
+}

infrastructure/environment/prod/variable.tf

@@ -0,0 +1,10 @@
+variable "project_id" {
+  type        = string
+  description = "google cloud project id"
+}
+
+variable "region" {
+  type        = string
+  default     = "us-central1"
+  description = "google cloud region"
+}

infrastructure/environment/staging/README.md

@@ -0,0 +1,23 @@
+# Trigger for node-pack-extract production
+
+Terraform modules to setup trigger for cloud build that will run [node-pack-extract](../../../node-pack-extract/)
+
+## Requirements
+
+- Google Cloud Account
+
+## Configuration
+
+This use the following configuration value:
+
+- bucket_name: "comfy-registry "
+- service account: "<[email protected]>"
+- topic_name: "comfy-registry-event-staging"
+
+## Apply
+
+```bash
+terraform apply 
+    -var project_id=dreamboothy-dev
+    -var region=us-central1
+```

infrastructure/environment/staging/main.tf

@@ -0,0 +1,24 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "6.14.1"
+    }
+  }
+}
+
+provider "google" {
+  project = var.project_id
+  region  = var.region
+}
+
+module "node_pack_extract_trigger" {
+  source = "../../modules/node-pack-extract-trigger"
+  providers = {
+    google = google
+  }
+  region                      = var.region
+  bucket_name                 = "comfy-registry "
+  cloud_build_service_account = "[email protected]"
+  topic_name                  = "comfy-registry-event-stage"
+}

infrastructure/environment/staging/variable.tf

@@ -0,0 +1,10 @@
+variable "project_id" {
+  type        = string
+  description = "google cloud project id"
+}
+
+variable "region" {
+  type        = string
+  default     = "us-central1"
+  description = "google cloud region"
+}

infrastructure/modules/node-pack-extract-trigger/README.md

@@ -0,0 +1,9 @@
+# Trigger for node-pack-extract
+
+Terraform modules to setup trigger for cloud build that will run [node-pack-extract](../../../node-pack-extract/)
+
+## Requirements
+
+- Google Cloud Account
+- Existing Google Cloud Storage bucket where the Registry backend store the comfy node packs.
+- Existing Service Account with `Service Account Token Creator` Role plus the one that is whitelisted in [service_account_auth](../../../server/middleware/authentication/service_account_auth.go#65) middleware.

infrastructure/modules/node-pack-extract-trigger/main.tf

@@ -0,0 +1,67 @@
+# get the existing GCS bucket
+data "google_storage_bucket" "bucket" {
+  name = var.bucket_name
+}
+
+# create a Pub/Sub topic
+resource "google_pubsub_topic" "topic" {
+  name = var.topic_name
+}
+
+# get the default GCS service account
+data "google_storage_project_service_account" "gcs_account" {
+}
+
+# Grant the GCS service account permission to publish to the Pub/Sub topic
+resource "google_pubsub_topic_iam_binding" "gcs_publisher" {
+  topic   = google_pubsub_topic.topic.name
+  role    = "roles/pubsub.publisher"
+  members = ["serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"]
+}
+
+# enable GCS Bucket Notification to Pub/Sub
+resource "google_storage_notification" "notification" {
+  bucket         = data.google_storage_bucket.bucket.name
+  topic          = google_pubsub_topic.topic.id
+  payload_format = "JSON_API_V1"
+  depends_on     = [google_pubsub_topic_iam_binding.gcs_publisher]
+  event_types = [
+    "OBJECT_FINALIZE", # Triggered when an object is successfully created or overwritten
+  ]
+}
+
+
+# Get the existing cloudbuild service account
+data "google_service_account" "cloudbuild_service_account" {
+  account_id = var.cloud_build_service_account
+}
+
+# Create the cloud build trigger
+resource "google_cloudbuild_trigger" "trigger" {
+  name            = "comfy-registry-nodepack"
+  location        = var.region
+  service_account = data.google_service_account.cloudbuild_service_account.id
+
+  pubsub_config {
+    topic = google_pubsub_topic.topic.id
+  }
+
+  source_to_build {
+    uri       = var.git_repo_uri
+    ref       = "refs/heads/${var.git_repo_branch}"
+    repo_type = "GITHUB"
+  }
+
+  git_file_source {
+    uri       = var.git_repo_uri
+    revision  = "refs/heads/${var.git_repo_branch}"
+    repo_type = "GITHUB"
+    path      = "node-pack-extract/cloudbuild.yaml"
+  }
+
+  substitutions = {
+    _CUSTOM_NODE_NAME = "custom-node"
+    _CUSTOM_NODE_URL  = "https://storage.googleapis.com/$(body.message.data.bucket)/$(body.message.data.name)"
+    _NODEPACK_HANDLER = var.registry_backend_url
+  }
+}

infrastructure/modules/node-pack-extract-trigger/variable.tf

@@ -0,0 +1,42 @@
+# REQUIRED VARIABLE
+variable "bucket_name" {
+  type        = string
+  description = "existing bucket name"
+}
+
+variable "cloud_build_service_account" {
+  type        = string
+  description = "Existing service account used to run the cloud build and used to access registry backend, e.g. [email protected]. Note that this service account needs to have 'Service Account Token Creator' role."
+}
+
+# OPTIONAL VARIABLE
+variable "region" {
+  type        = string
+  default     = "us-central1"
+  description = "google cloud region"
+}
+
+variable "topic_name" {
+  type        = string
+  description = "pub/sub topic to be created"
+  default     = "comfy-registry-event"
+}
+
+variable "git_repo_uri" {
+  type        = string
+  description = "git repo containing the cloud build pipeline"
+  default     = "https://github.com/Comfy-Org/registry-backend"
+}
+
+variable "git_repo_branch" {
+  type        = string
+  description = "git repo branch"
+  default     = "master"
+}
+
+variable "registry_backend_url" {
+  type        = string
+  description = "the url where registry backend can be reached"
+  default     = "https://api.comfy.org"
+}
+

infrastructure/modules/node-pack-extract-trigger/version.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "6.14.1"
+    }
+  }
+}

node-pack-extract/Dockerfile

@@ -0,0 +1,8 @@
+FROM ghcr.io/ai-dock/comfyui:v2-cpu-22.04-v0.2.7
+
+COPY ./provisioning.sh /opt/ai-dock/bin/provisioning.sh
+COPY ./entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENV WEB_ENABLE_AUTH false
+ENTRYPOINT [ "/entrypoint.sh" ]

node-pack-extract/README.md

@@ -0,0 +1,13 @@
+# Generate ComfyUI Node Pack Information
+
+Execute the following command. You can adjust `CUSTOM_NODE_URL` and `CUSTOM_NODE_NAME`.
+
+```bash
+docker compose run node-pack-extact \
+    --build \
+    -e CUSTOM_NODE_URL=https://storage.googleapis.com/comfy-registry/altkeyproject/comfyui-dream-project/1.0.6/node.tar.gz \
+    -e CUSTOM_NODE_NAME=comfyui-dream-project \
+    comfyui-dream-project-nodepack.json
+```
+
+The command will produce `comfyui-dream-project-nodepack.json` under [`output`](./output/) directory.

node-pack-extract/cloudbuild.yaml

@@ -0,0 +1,46 @@
+steps:
+  - name: "gcr.io/cloud-builders/docker"
+    args:
+      - "build"
+      - "-t"
+      - "gcr.io/$PROJECT_ID/comfyui-nodepack"
+      - "."
+
+  - name: "gcr.io/cloud-builders/docker"
+    args:
+      - "run"
+      - "--env"
+      - "CUSTOM_NODE_URL=$_CUSTOM_NODE_URL"
+      - "--env"
+      - "CUSTOM_NODE_NAME=$_CUSTOM_NODE_NAME"
+      - "--volume"
+      - "/workspace:/workspace"
+      - "--workdir"
+      - "/workspace"
+      - "gcr.io/$PROJECT_ID/comfyui-nodepack"
+      - "$_CUSTOM_NODE_NAME.json"
+
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk:slim"
+    entrypoint: "bash"
+    args:
+      - -c 
+      - gcloud auth print-identity-token | tee /workspace/token
+
+  - name: "gcr.io/cloud-builders/curl"
+    entrypoint: "bash"
+    args:
+      - -c
+      - |
+        curl -s \
+          -H "Authorization: Bearer $(cat /workspace/token)" \
+          -H "Content-Type: application/json" \
+          -X "POST" "-d" "@/workspace/$_CUSTOM_NODE_NAME.json" \
+          "$_NODEPACK_HANDLER"
+
+substitutions:
+  _CUSTOM_NODE_NAME: "default-node-name"
+  _CUSTOM_NODE_URL: "https://example.com/default-node.tar.gz"
+  _NODEPACK_HANDLER: "https://ntfy.sh/comfyui-nodepack-handler"
+
+options:
+  logging: CLOUD_LOGGING_ONLY

node-pack-extract/docker-compose.yml

@@ -0,0 +1,10 @@
+services:
+  node-pack-extact:
+    build: .
+    environment:
+      CUSTOM_NODE_URL: https://storage.googleapis.com/comfy-registry/altkeyproject/comfyui-dream-project/1.0.6/node.tar.gz
+      CUSTOM_NODE_NAME: comfyui-dream-project
+    volumes:
+      - ./output:/tmp/output
+    working_dir: /tmp/output
+    command: comfyui-dream-project.json