diff --git a/.gitignore b/.gitignore index cf6ef10..f22de51 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,8 @@ tmp/ # Lazy way to make atlas usable on windows atlas.exe -.env + +# terraform +**/.terraform/* +terraform.tfstate +terraform.tfstate.* \ No newline at end of file diff --git a/infrastructure/environment/prod/.terraform.lock.hcl b/infrastructure/environment/prod/.terraform.lock.hcl new file mode 100644 index 0000000..8a45359 --- /dev/null +++ b/infrastructure/environment/prod/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "6.14.1" + constraints = "6.14.1" + hashes = [ + "h1:zhyWKVVqORklT7c28f6FzZ0z/g6pGa6FFtv/wp1MKDc=", + "zh:0a0cab3291bdac20fe31511b7aa9f3258b14add16d13110d4ebac18761277361", + "zh:178594db6fbff9974a7c65c65195a64c93d16f652a1a4136015b192faaa1ce2d", + "zh:379bbd6bd5b8add55ffd46c99a8081664e9004188f6df91f8f044e4268b86e42", + "zh:4899b6174a4492dbff3d94f56a901692a3f8d86a6db9de6a92b83d43b7ad4507", + "zh:6240820c3aeeaa8b9830fb4514d3ecb6e3fed8724340dfedaf89b4bb2265102f", + "zh:9a214e052c5c7b4e7bc409086832d4bd8e404b652b66344c26c314747c49744f", + "zh:a6cb9da102d371a52b750de5628a2b6b7cc7c20481d7fa3fb25a957b58b05777", + "zh:ab69d69f9c16461105a585f8a9c780eb06238827db269c5ad3f7c114922e7f20", + "zh:b716e219779295e9af2b632b2c0d534c90cca5c87352fc0c6dc8ebae671dd4d2", + "zh:d96e8d6c3109882d5527956acad12015a024fc24a6110e244007209b4f3069a0", + "zh:f49274564725189932a65cf1a4e8f40f46fca2537da7a673da9fea6ff7e3f195", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/infrastructure/environment/prod/README.md b/infrastructure/environment/prod/README.md new file mode 100644 index 0000000..10d50c4 --- /dev/null +++ b/infrastructure/environment/prod/README.md @@ -0,0 +1,23 @@ +# Trigger for node-pack-extract production + +Terraform modules to setup trigger for cloud build that will run [node-pack-extract](../../../node-pack-extract/) + +## Requirements + +- Google Cloud Account + +## Configuration + +This use the following configuration value: + +- bucket_name: "comfy-registry " +- service account: "" +- topic_name: "comfy-registry-event" + +## Apply + +```bash +terraform apply + -var project_id=dreamboothy-dev + -var region=us-central1 +``` diff --git a/infrastructure/environment/prod/main.tf b/infrastructure/environment/prod/main.tf new file mode 100644 index 0000000..6086d7e --- /dev/null +++ b/infrastructure/environment/prod/main.tf @@ -0,0 +1,24 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "6.14.1" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} + +module "node_pack_extract_trigger" { + source = "../../modules/node-pack-extract-trigger" + providers = { + google = google + } + region = var.region + bucket_name = "comfy-registry " + cloud_build_service_account = "cloud-scheduler@dreamboothy.iam.gserviceaccount.com" + topic_name = "comfy-registry-event" +} diff --git a/infrastructure/environment/prod/variable.tf b/infrastructure/environment/prod/variable.tf new file mode 100644 index 0000000..7694c45 --- /dev/null +++ b/infrastructure/environment/prod/variable.tf @@ -0,0 +1,10 @@ +variable "project_id" { + type = string + description = "google cloud project id" +} + +variable "region" { + type = string + default = "us-central1" + description = "google cloud region" +} diff --git a/infrastructure/environment/staging/README.md b/infrastructure/environment/staging/README.md new file mode 100644 index 0000000..3279ab1 --- /dev/null +++ b/infrastructure/environment/staging/README.md @@ -0,0 +1,23 @@ +# Trigger for node-pack-extract production + +Terraform modules to setup trigger for cloud build that will run [node-pack-extract](../../../node-pack-extract/) + +## Requirements + +- Google Cloud Account + +## Configuration + +This use the following configuration value: + +- bucket_name: "comfy-registry " +- service account: "" +- topic_name: "comfy-registry-event-staging" + +## Apply + +```bash +terraform apply + -var project_id=dreamboothy-dev + -var region=us-central1 +``` diff --git a/infrastructure/environment/staging/main.tf b/infrastructure/environment/staging/main.tf new file mode 100644 index 0000000..f0420f9 --- /dev/null +++ b/infrastructure/environment/staging/main.tf @@ -0,0 +1,24 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "6.14.1" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} + +module "node_pack_extract_trigger" { + source = "../../modules/node-pack-extract-trigger" + providers = { + google = google + } + region = var.region + bucket_name = "comfy-registry " + cloud_build_service_account = "cloud-scheduler@dreamboothy.iam.gserviceaccount.com" + topic_name = "comfy-registry-event-stage" +} diff --git a/infrastructure/environment/staging/variable.tf b/infrastructure/environment/staging/variable.tf new file mode 100644 index 0000000..7694c45 --- /dev/null +++ b/infrastructure/environment/staging/variable.tf @@ -0,0 +1,10 @@ +variable "project_id" { + type = string + description = "google cloud project id" +} + +variable "region" { + type = string + default = "us-central1" + description = "google cloud region" +} diff --git a/infrastructure/modules/node-pack-extract-trigger/.terraform.lock.hcl b/infrastructure/modules/node-pack-extract-trigger/.terraform.lock.hcl new file mode 100644 index 0000000..8a45359 --- /dev/null +++ b/infrastructure/modules/node-pack-extract-trigger/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "6.14.1" + constraints = "6.14.1" + hashes = [ + "h1:zhyWKVVqORklT7c28f6FzZ0z/g6pGa6FFtv/wp1MKDc=", + "zh:0a0cab3291bdac20fe31511b7aa9f3258b14add16d13110d4ebac18761277361", + "zh:178594db6fbff9974a7c65c65195a64c93d16f652a1a4136015b192faaa1ce2d", + "zh:379bbd6bd5b8add55ffd46c99a8081664e9004188f6df91f8f044e4268b86e42", + "zh:4899b6174a4492dbff3d94f56a901692a3f8d86a6db9de6a92b83d43b7ad4507", + "zh:6240820c3aeeaa8b9830fb4514d3ecb6e3fed8724340dfedaf89b4bb2265102f", + "zh:9a214e052c5c7b4e7bc409086832d4bd8e404b652b66344c26c314747c49744f", + "zh:a6cb9da102d371a52b750de5628a2b6b7cc7c20481d7fa3fb25a957b58b05777", + "zh:ab69d69f9c16461105a585f8a9c780eb06238827db269c5ad3f7c114922e7f20", + "zh:b716e219779295e9af2b632b2c0d534c90cca5c87352fc0c6dc8ebae671dd4d2", + "zh:d96e8d6c3109882d5527956acad12015a024fc24a6110e244007209b4f3069a0", + "zh:f49274564725189932a65cf1a4e8f40f46fca2537da7a673da9fea6ff7e3f195", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/infrastructure/modules/node-pack-extract-trigger/README.md b/infrastructure/modules/node-pack-extract-trigger/README.md new file mode 100644 index 0000000..c4c9016 --- /dev/null +++ b/infrastructure/modules/node-pack-extract-trigger/README.md @@ -0,0 +1,9 @@ +# Trigger for node-pack-extract + +Terraform modules to setup trigger for cloud build that will run [node-pack-extract](../../../node-pack-extract/) + +## Requirements + +- Google Cloud Account +- Existing Google Cloud Storage bucket where the Registry backend store the comfy node packs. +- Existing Service Account with `Service Account Token Creator` Role plus the one that is whitelisted in [service_account_auth](../../../server/middleware/authentication/service_account_auth.go#65) middleware. diff --git a/infrastructure/modules/node-pack-extract-trigger/main.tf b/infrastructure/modules/node-pack-extract-trigger/main.tf new file mode 100644 index 0000000..2dbffc4 --- /dev/null +++ b/infrastructure/modules/node-pack-extract-trigger/main.tf @@ -0,0 +1,67 @@ +# get the existing GCS bucket +data "google_storage_bucket" "bucket" { + name = var.bucket_name +} + +# create a Pub/Sub topic +resource "google_pubsub_topic" "topic" { + name = var.topic_name +} + +# get the default GCS service account +data "google_storage_project_service_account" "gcs_account" { +} + +# Grant the GCS service account permission to publish to the Pub/Sub topic +resource "google_pubsub_topic_iam_binding" "gcs_publisher" { + topic = google_pubsub_topic.topic.name + role = "roles/pubsub.publisher" + members = ["serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"] +} + +# enable GCS Bucket Notification to Pub/Sub +resource "google_storage_notification" "notification" { + bucket = data.google_storage_bucket.bucket.name + topic = google_pubsub_topic.topic.id + payload_format = "JSON_API_V1" + depends_on = [google_pubsub_topic_iam_binding.gcs_publisher] + event_types = [ + "OBJECT_FINALIZE", # Triggered when an object is successfully created or overwritten + ] +} + + +# Get the existing cloudbuild service account +data "google_service_account" "cloudbuild_service_account" { + account_id = var.cloud_build_service_account +} + +# Create the cloud build trigger +resource "google_cloudbuild_trigger" "trigger" { + name = "comfy-registry-nodepack" + location = var.region + service_account = data.google_service_account.cloudbuild_service_account.id + + pubsub_config { + topic = google_pubsub_topic.topic.id + } + + source_to_build { + uri = var.git_repo_uri + ref = "refs/heads/${var.git_repo_branch}" + repo_type = "GITHUB" + } + + git_file_source { + uri = var.git_repo_uri + revision = "refs/heads/${var.git_repo_branch}" + repo_type = "GITHUB" + path = "node-pack-extract/cloudbuild.yaml" + } + + substitutions = { + _CUSTOM_NODE_NAME = "custom-node" + _CUSTOM_NODE_URL = "https://storage.googleapis.com/$(body.message.data.bucket)/$(body.message.data.name)" + _NODEPACK_HANDLER = var.registry_backend_url + } +} diff --git a/infrastructure/modules/node-pack-extract-trigger/variable.tf b/infrastructure/modules/node-pack-extract-trigger/variable.tf new file mode 100644 index 0000000..ceb77b0 --- /dev/null +++ b/infrastructure/modules/node-pack-extract-trigger/variable.tf @@ -0,0 +1,42 @@ +# REQUIRED VARIABLE +variable "bucket_name" { + type = string + description = "existing bucket name" +} + +variable "cloud_build_service_account" { + type = string + description = "Existing service account used to run the cloud build and used to access registry backend, e.g. cloud-build@my-project.iam.gserviceaccount.com. Note that this service account needs to have 'Service Account Token Creator' role." +} + +# OPTIONAL VARIABLE +variable "region" { + type = string + default = "us-central1" + description = "google cloud region" +} + +variable "topic_name" { + type = string + description = "pub/sub topic to be created" + default = "comfy-registry-event" +} + +variable "git_repo_uri" { + type = string + description = "git repo containing the cloud build pipeline" + default = "https://github.com/Comfy-Org/registry-backend" +} + +variable "git_repo_branch" { + type = string + description = "git repo branch" + default = "master" +} + +variable "registry_backend_url" { + type = string + description = "the url where registry backend can be reached" + default = "https://api.comfy.org" +} + diff --git a/infrastructure/modules/node-pack-extract-trigger/version.tf b/infrastructure/modules/node-pack-extract-trigger/version.tf new file mode 100644 index 0000000..7818333 --- /dev/null +++ b/infrastructure/modules/node-pack-extract-trigger/version.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "6.14.1" + } + } +} diff --git a/node-pack-extract/Dockerfile b/node-pack-extract/Dockerfile new file mode 100644 index 0000000..4cfcc41 --- /dev/null +++ b/node-pack-extract/Dockerfile @@ -0,0 +1,8 @@ +FROM ghcr.io/ai-dock/comfyui:v2-cpu-22.04-v0.2.7 + +COPY ./provisioning.sh /opt/ai-dock/bin/provisioning.sh +COPY ./entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENV WEB_ENABLE_AUTH false +ENTRYPOINT [ "/entrypoint.sh" ] \ No newline at end of file diff --git a/node-pack-extract/README.md b/node-pack-extract/README.md new file mode 100644 index 0000000..a15a180 --- /dev/null +++ b/node-pack-extract/README.md @@ -0,0 +1,13 @@ +# Generate ComfyUI Node Pack Information + +Execute the following command. You can adjust `CUSTOM_NODE_URL` and `CUSTOM_NODE_NAME`. + +```bash +docker compose run node-pack-extact \ + --build \ + -e CUSTOM_NODE_URL=https://storage.googleapis.com/comfy-registry/altkeyproject/comfyui-dream-project/1.0.6/node.tar.gz \ + -e CUSTOM_NODE_NAME=comfyui-dream-project \ + comfyui-dream-project-nodepack.json +``` + +The command will produce `comfyui-dream-project-nodepack.json` under [`output`](./output/) directory. diff --git a/node-pack-extract/cloudbuild.yaml b/node-pack-extract/cloudbuild.yaml new file mode 100644 index 0000000..2021dc6 --- /dev/null +++ b/node-pack-extract/cloudbuild.yaml @@ -0,0 +1,46 @@ +steps: + - name: "gcr.io/cloud-builders/docker" + args: + - "build" + - "-t" + - "gcr.io/$PROJECT_ID/comfyui-nodepack" + - "." + + - name: "gcr.io/cloud-builders/docker" + args: + - "run" + - "--env" + - "CUSTOM_NODE_URL=$_CUSTOM_NODE_URL" + - "--env" + - "CUSTOM_NODE_NAME=$_CUSTOM_NODE_NAME" + - "--volume" + - "/workspace:/workspace" + - "--workdir" + - "/workspace" + - "gcr.io/$PROJECT_ID/comfyui-nodepack" + - "$_CUSTOM_NODE_NAME.json" + + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk:slim" + entrypoint: "bash" + args: + - -c + - gcloud auth print-identity-token | tee /workspace/token + + - name: "gcr.io/cloud-builders/curl" + entrypoint: "bash" + args: + - -c + - | + curl -s \ + -H "Authorization: Bearer $(cat /workspace/token)" \ + -H "Content-Type: application/json" \ + -X "POST" "-d" "@/workspace/$_CUSTOM_NODE_NAME.json" \ + "$_NODEPACK_HANDLER" + +substitutions: + _CUSTOM_NODE_NAME: "default-node-name" + _CUSTOM_NODE_URL: "https://example.com/default-node.tar.gz" + _NODEPACK_HANDLER: "https://ntfy.sh/comfyui-nodepack-handler" + +options: + logging: CLOUD_LOGGING_ONLY \ No newline at end of file diff --git a/node-pack-extract/docker-compose.yml b/node-pack-extract/docker-compose.yml new file mode 100644 index 0000000..62615f0 --- /dev/null +++ b/node-pack-extract/docker-compose.yml @@ -0,0 +1,10 @@ +services: + node-pack-extact: + build: . + environment: + CUSTOM_NODE_URL: https://storage.googleapis.com/comfy-registry/altkeyproject/comfyui-dream-project/1.0.6/node.tar.gz + CUSTOM_NODE_NAME: comfyui-dream-project + volumes: + - ./output:/tmp/output + working_dir: /tmp/output + command: comfyui-dream-project.json diff --git a/node-pack-extract/entrypoint.sh b/node-pack-extract/entrypoint.sh new file mode 100644 index 0000000..650c12e --- /dev/null +++ b/node-pack-extract/entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -e + +# run the original endpoint +init.sh & + +# loop until we can extract the node information +OUTPUTFILE=${1:-"/tmp/output.json"} +echo -n > "$OUTPUTFILE" +until cat "$OUTPUTFILE" | grep ''; do + curl -sf localhost:8188/object_info | + jq -c ' + to_entries | + map( + select(.value.python_module == "custom_nodes.'$CUSTOM_NODE_NAME'") + | .value |= { + category : .category, + description : .description, + deprecated : .deprecated, + experimental : .experimental, + input_types : (.input | tojson), + return_names : .output_name, + return_types : .output, + output_is_list : .output_is_list, + } + ) | + if length > 0 then from_entries else "" end' | + tee "$OUTPUTFILE" + + sleep 1 +done + +# make sure its json or we fail +grep '{' "$OUTPUTFILE" diff --git a/node-pack-extract/provisioning.sh b/node-pack-extract/provisioning.sh new file mode 100644 index 0000000..bec9342 --- /dev/null +++ b/node-pack-extract/provisioning.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +echo "===========================================" +echo "=== Downloading the custom node archive ===" +echo "===========================================" +cd /opt/ComfyUI/custom_nodes/ +rm -rf "$CUSTOM_NODE_NAME" +mkdir -p "$CUSTOM_NODE_NAME" +wget -O "$CUSTOM_NODE_NAME.zip" "$CUSTOM_NODE_URL" + +echo "==================================" +echo "=== Installing the custom node ===" +echo "==================================" +unzip "${CUSTOM_NODE_NAME}.zip" -d "$CUSTOM_NODE_NAME" +cd "$CUSTOM_NODE_NAME" +source /opt/environments/python/comfyui/bin/activate +pip install -r requirements.txt +if [ -f "install.py" ]; then + python install.py +fi