HuaweiCloud-Kafka-terraform_module.yml

# The version of the Xpanse description language
version: 1.0
# The category of the service.
category: middleware
# The Service provided by the ISV, the name will be shown on the console as a service.
name: Kafka-cluster
# The version of the service, the end-user can select the version they want to deploy.
serviceVersion: 1.0.0
# For the users may have more than one service, the @namespace can be used to separate the clusters.
description: This is an enhanced Kafka cluster services by ISV-A.
namespace: ISV-A
# Icon for the service.
icon: |
  data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAACRAQMAAAAPc4+9AAAAAXNSR0IB2cksfwAAAAlwSFlzAAALEwAACxMBAJqcGAAAAAZQTFRF+/v7Hh8gVD0A0wAAAcVJREFUeJzNlc1twzAMhSX44KNH0CgaTd6gK3kUd4McDVTwq/hjiUyaIkV7qNA2/QCFIh+ppxB+svLNEqqBGTC0ANugBOwmCGDCFOAwIWGDOoqoODtN2BdL6wxD9NMTO9tXPa1PqL5M30W5p8lm5vNcF0t7ahSrVguqNqmMokRW4YQucVjBCBWH1Z2g3WDlW2skoYU+2x8JOtGedBF3k2iXMO0j16iUiI6gxzPdQhnU/s2G9pCO57QY2r6hvjPbKJHq7DRTRXT60avtuTRdbrFJI3mSZhNOqYjVbd99YyK1QKWzEqSWrE0k07U60uPaelflMzaaeu1KBuurHSsn572I1KWy2joX5ZBfWbS/VEt50H5P6aL4JxTuyJ/+QCNPX4PWF3Q8Xe1eF9FsLdD2VaOnaP2hWvs+zI58/7i3vH3nRFtDZpyTUNaZkON5XnBNsp8lrmDMrpvBr+b6pUl+4XbkQdndqnzYGzfuJm1JmIWimIbe6dndd/bk7gVce/cJdo3uIeLJl7+I2xTnPek67mjtDeppE7b03Ov+kSfDe3JweW53njxeGfXkaz28VeYd86+af/H8a7hgJKaebILaFzakLfxyfQLTxVB6K1K9KQAAAABJRU5ErkJggg==
# Reserved for CSP: HuaweiCloud,FlexibleEngine,OpenstackTestlab,PlusServer,RegioCloud and ...
cloudServiceProvider:
  name: HuaweiCloud
  regions:
    - name: cn-southwest-2
      area: Asia China
    - name: cn-north-4
      area: Asia China
billing:
  # The supported mode of billing (`Fixed`, `Pay per Use`)
  billingModes:
    - Fixed
    - Pay per Use
serviceHostingType: self
# The flavor of the service, the @category/@name/@version/@flavor can locate the specific service to be deployed.
flavors:
  serviceFlavors:
    - name: 1-zookeeper-with-3-worker-nodes-normal
      priority: 2
      # The pricing of the flavor.
      pricing:
        # Used to calculate charges when users select 'pay_per_use' as the billing mode.
        resourceUsage:
          resources:
            - deployResourceKind: vm
              count: 4
              properties:
                cloud_service_type: hws.service.type.ec2
                resource_type: hws.resource.type.vm
                resource_spec: s6.large.2.linux
            - deployResourceKind: volume
              count: 4
              properties:
                cloud_service_type: hws.service.type.ebs
                resource_type: hws.resource.type.volume
                resource_spec: SSD
                resource_size: 40
                size_measure_id: 17
          licensePrice:
            cost: 1.90
            currency: CNY
            period: hourly
          markUpPrice:
            cost: 1.60
            currency: CNY
            period: hourly
        # Used to calculate charges when users do not select 'pay_per_use' as the billing mode.
        fixedPrice:
          cost: 730
          currency: CNY
          period: monthly
      # Properties for the service, which can be used by the deployment.
      properties:
        worker_nodes_count: 3
        flavor_id: s6.large.2
      features:
        - High Availability
        - Maximum performance
    - name: 1-zookeeper-with-3-worker-nodes-performance
      priority: 1
      # The pricing of the flavor.
      pricing:
        # Used to calculate charges when users select 'pay_per_use' as the billing mode.
        resourceUsage:
          resources:
            - deployResourceKind: vm
              count: 4
              properties:
                cloud_service_type: hws.service.type.ec2
                resource_type: hws.resource.type.vm
                resource_spec: s6.large.4.linux
            - deployResourceKind: volume
              count: 4
              properties:
                cloud_service_type: hws.service.type.ebs
                resource_type: hws.resource.type.volume
                resource_spec: SSD
                resource_size: 40
                size_measure_id: 17
          licensePrice:
            cost: 3.20
            currency: CNY
            period: hourly
          markUpPrice:
            cost: 2.88
            currency: CNY
            period: hourly
        # Used to calculate charges when users do not select 'pay_per_use' as the billing mode.
        fixedPrice:
          cost: 980
          currency: CNY
          period: monthly
      properties:
        worker_nodes_count: 3
        flavor_id: s6.large.4
      features:
        - High Availability
        - Maximum performance
  modificationImpact:
    isDataLost: true
    isServiceInterrupted: true
  isDowngradeAllowed: true
# The contact details of the service.
serviceProviderContactDetails:
  emails: [ "test@test.com" ]
configurationParameters:
  - name: kafka_cfg_message_max_bytes
    kind: variable
    dataType: number
    description: Parameters used to configure the Kafka server or client to set the maximum number of bytes for Kafka messages.
    initialValue: 1048576
    modificationImpact:
      isDataLost: true
      isServiceInterrupted: true
    isReadOnly: false
  - name: kafka_cfg_log_dirs
    kind: variable
    dataType: string
    description: Parameters for the storage location of Kafka log data
    initialValue: /var/lib/kafka/logs
    modificationImpact:
      isDataLost: true
      isServiceInterrupted: true
    isReadOnly: false
  - name: kafka_cfg_num_io_threads
    kind: variable
    dataType: number
    description: Parameter used to set the number of I/O threads to handle kafka network requests
    initialValue: 8
    modificationImpact:
      isDataLost: true
      isServiceInterrupted: true
    isReadOnly: false
  - name: kafka_log_flush_interval_messages
    kind: variable
    dataType: number
    description: Kafka's log flush strategy, specifies the parameters that trigger a log flush operation after how many messages are written.
    initialValue: 10000
    modificationImpact:
      isDataLost: true
      isServiceInterrupted: true
    isReadOnly: false
  - name: kafka_offsets_topic_replication_factor
    kind: variable
    dataType: number
    description: Replication factor configuration parameters for the dedicated topic (offsets topic) used in the Kafka cluster to consume offset information for the consumer storage group
    initialValue: 3
    modificationImpact:
      isDataLost: true
      isServiceInterrupted: true
    isReadOnly: false
deployment:
  # kind, Supported values are terraform, opentofu.
  kind: terraform
  serviceAvailabilityConfigs:
    - displayName: Availability Zone
      varName: availability_zone
      mandatory: false
      description: The availability zone to deploy the service instance. If the value is empty, the service instance will be deployed in a random availability zone.
  # Context for deployment: the context including some kind of parameters for the deployment, such as fix_env, fix_variable, env, variable, env_env, env_variable.
  # - fix_env: Values for variable of this type are defined by the managed service provider in the OCL template. Runtime will inject it to deployer as environment variables. This variable is not visible to the end user.
  # - fix_variable: Values for variable of this type are defined by the managed service provider in the OCL template. Runtime will inject it to deployer as usual variables. This variable is not visible to the end user.
  # - env: Value for a variable of this type can be provided by end user. If marked as mandatory then end user must provide value to this variable. If marked as optional and if end user does not provided it, then the fallback value to this variable is read by runtime (it can read from other sources, e.g., OS env variables). This variable is injected as a environment variable to the deployer.
  # - variable: Value for a variable of this type can be provided by end user. . If marked as mandatory then end user must provide value to this variable. If marked as optional and if end user does not provided it, then the fallback value to this variable is read by runtime (it can read from other sources, e.g., OS env variables). This variable is injected as a regular variable to the deployer.
  # - env_env: Value to this variable is read by runtime (it can read from other sources, e.g., OS env variables) and injected as a environment variable to the deployer. End user cannot see or change this variable.
  # - env_variable: Value to this variable is read by runtime (it can read from other sources, e.g., OS env variables) and injected as a regular variable to the deployer. End user cannot see or change this variable.
  # The parameters will be used to generate the API of the managed service.
  variables:
    - name: admin_passwd
      description: The admin password of all nodes in the Kafka cluster instance. If the value is empty, will create a random password.
      kind: variable
      dataType: string
      mandatory: false
      valueSchema:
        minLength: 8
        maxLength: 16
        pattern: ^(?=.*?[A-Z])(?=.*?[a-z])(?=.*?[0-9])(?=.*?[#?!@$%^&*-]).{8,16}$
      modificationImpact:
        isDataLost: true
        isServiceInterrupted: true
    - name: vpc_name
      description: The vpc name of all nodes in the Kafka cluster instance. If the value is empty, will use the example value to find or create VPC.
      kind: variable
      dataType: string
      example: "kafka-vpc-default"
      mandatory: false
      modificationImpact:
        isDataLost: true
        isServiceInterrupted: true
    - name: subnet_name
      description: The sub network name of all nodes in the Kafka cluster instance. If the value is empty, will use the example value to find or create subnet.
      kind: variable
      dataType: string
      example: "kafka-subnet-default"
      mandatory: false
      modificationImpact:
        isDataLost: true
        isServiceInterrupted: true
    - name: secgroup_name
      description: The security group name of all nodes in the Kafka cluster instance. If the value is empty, will use the example value to find or create security group.
      kind: variable
      dataType: string
      example: "kafka-secgroup-default"
      mandatory: false
      modificationImpact:
        isDataLost: true
        isServiceInterrupted: true
  deployer: |
    variable "region" {
      type        = string
      description = "The region to deploy the Kafka cluster instance."
    }
    
    variable "availability_zone" {
      type        = string
      default     = ""
      description = "The availability zone to deploy the Kafka cluster instance."
    }
    
    variable "flavor_id" {
      type        = string
      default     = "s6.large.2"
      description = "The flavor_id of all nodes in the Kafka cluster instance."
    }

    variable "worker_nodes_count" {
      type        = string
      default     = 3
      description = "The worker nodes count in the Kafka cluster instance."
    }

    variable "admin_passwd" {
      type        = string
      default     = ""
      description = "The root password of all nodes in the Kafka cluster instance."
    }

    variable "vpc_name" {
      type        = string
      default     = "kafka-vpc-default"
      description = "The vpc name of all nodes in the Kafka cluster instance."
    }

    variable "subnet_name" {
      type        = string
      default     = "kafka-subnet-default"
      description = "The subnet name of all nodes in the Kafka cluster instance."
    }

    variable "secgroup_name" {
      type        = string
      default     = "kafka-secgroup-default"
      description = "The security group name of all nodes in the Kafka cluster instance."
    }
    
    terraform {
      required_providers {
        huaweicloud = {
          source  = "huaweicloud/huaweicloud"
          version = "~> 1.61.0"
        }
      }
    }
    
    provider "huaweicloud" {
      region = var.region
    }
    
    data "huaweicloud_availability_zones" "osc-az" {}

    data "huaweicloud_vpcs" "existing" {
      name = var.vpc_name
    }

    data "huaweicloud_vpc_subnets" "existing" {
      name = var.subnet_name
    }

    data "huaweicloud_networking_secgroups" "existing" {
      name = var.secgroup_name
    }

    locals {
      availability_zone = var.availability_zone == "" ? data.huaweicloud_availability_zones.osc-az.names[0] : var.availability_zone
      admin_passwd      = var.admin_passwd == "" ? random_password.password.result : var.admin_passwd
      vpc_id            = length(data.huaweicloud_vpcs.existing.vpcs) > 0 ? data.huaweicloud_vpcs.existing.vpcs[0].id : huaweicloud_vpc.new[0].id
      subnet_id         = length(data.huaweicloud_vpc_subnets.existing.subnets)> 0 ? data.huaweicloud_vpc_subnets.existing.subnets[0].id : huaweicloud_vpc_subnet.new[0].id
      secgroup_id       = length(data.huaweicloud_networking_secgroups.existing.security_groups) > 0 ? data.huaweicloud_networking_secgroups.existing.security_groups[0].id : huaweicloud_networking_secgroup.new[0].id
    }

    resource "huaweicloud_vpc" "new" {
      count = length(data.huaweicloud_vpcs.existing.vpcs) == 0 ? 1 : 0
      name  = var.vpc_name
      cidr  = "192.168.0.0/16"
    }

    resource "huaweicloud_vpc_subnet" "new" {
      count      = length(data.huaweicloud_vpcs.existing.vpcs) == 0 ? 1 : 0
      vpc_id     = local.vpc_id
      name       = var.subnet_name
      cidr       = "192.168.10.0/24"
      gateway_ip = "192.168.10.1"
    }

    resource "huaweicloud_networking_secgroup" "new" {
      count       = length(data.huaweicloud_networking_secgroups.existing.security_groups) == 0 ? 1 : 0
      name        = var.secgroup_name
      description = "Kafka cluster security group"
    }

    resource "huaweicloud_networking_secgroup_rule" "secgroup_rule_0" {
      count             = length(data.huaweicloud_networking_secgroups.existing.security_groups) == 0 ? 1 : 0
      direction         = "ingress"
      ethertype         = "IPv4"
      protocol          = "tcp"
      port_range_min    = 22
      port_range_max    = 22
      remote_ip_prefix  = "121.37.117.211/32"
      security_group_id = local.secgroup_id
    }

    resource "huaweicloud_networking_secgroup_rule" "secgroup_rule_1" {
      count             = length(data.huaweicloud_networking_secgroups.existing.security_groups) == 0 ? 1 : 0
      direction         = "ingress"
      ethertype         = "IPv4"
      protocol          = "tcp"
      port_range_min    = 2181
      port_range_max    = 2181
      remote_ip_prefix  = "121.37.117.211/32"
      security_group_id = local.secgroup_id
    }

    resource "huaweicloud_networking_secgroup_rule" "secgroup_rule_2" {
      count             = length(data.huaweicloud_networking_secgroups.existing.security_groups) == 0 ? 1 : 0
      direction         = "ingress"
      ethertype         = "IPv4"
      protocol          = "tcp"
      port_range_min    = 9092
      port_range_max    = 9093
      remote_ip_prefix  = "121.37.117.211/32"
      security_group_id = local.secgroup_id
    }

    resource "random_id" "new" {
      byte_length = 4
    }

    resource "random_password" "password" {
      length           = 12
      upper            = true
      lower            = true
      numeric          = true
      special          = true
      min_special      = 1
      override_special = "#%@"
    }

    resource "huaweicloud_compute_keypair" "keypair" {
      name = "keypair-kafka-${random_id.new.hex}"
    }

    data "huaweicloud_images_image" "image" {
      name                  = "Kafka-v3.3.2_Ubuntu-20.04"
      most_recent           = true
      enterprise_project_id = "0"
    }

    module "ecs_kafka_service" {
        source 			 = "git@github.com:terraform-huaweicloud-modules/terraform-huaweicloud-ecs.git"            
        count 			 = var.worker_nodes_count
        availability_zone        = local.availability_zone
        subnet_id                = local.subnet_id
        instance_name            = "kafka-broker-module${random_id.new.hex}-${count.index}"
        instance_flavor_id       = var.flavor_id
        instance_image_id        = data.huaweicloud_images_image.image.id
        security_group_ids       = [ local.secgroup_id ]
        system_disk_size         = 50
        admin_password           = local.admin_passwd
        keypair_name             = huaweicloud_compute_keypair.keypair.name       
        user_data = <<EOF
            #!bin/bash
            echo root:${local.admin_passwd} | sudo chpasswd
            sudo systemctl start docker
            sudo systemctl enable docker
            private_ip=$(ifconfig | grep -A1 "eth0" | grep 'inet' | awk -F ' ' ' {print $2}'|awk ' {print $1}')
            sudo docker run -d --name kafka-server --restart always -p 9092:9092 -p 9093:9093  -e KAFKA_BROKER_ID=${count.index}  -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://$private_ip:9092 -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 -e ALLOW_PLAINTEXT_LISTENER=yes -e KAFKA_CFG_ZOOKEEPER_CONNECT=${module.ecs_zookeeper_service.instance_access_ipv4}:2181 bitnami/kafka:3.3.2
        EOF
        depends_on = [
            module.ecs_zookeeper_service
        ]
    }
        
    module "ecs_zookeeper_service" {
        source 			 = "git@github.com:terraform-huaweicloud-modules/terraform-huaweicloud-ecs.git"    
        availability_zone        = local.availability_zone
        subnet_id          	 = local.subnet_id
        instance_name            = "kafka-zookeeper-module${random_id.new.hex}"
        instance_flavor_id       = var.flavor_id
        instance_image_id        = data.huaweicloud_images_image.image.id
        security_group_ids       = [ local.secgroup_id ]
        system_disk_size         = 50
        admin_password           = local.admin_passwd
        keypair_name             = huaweicloud_compute_keypair.keypair.name
        user_data = <<EOF
            #!bin/bash
            echo root:${local.admin_passwd} | sudo chpasswd
            sudo systemctl start docker
            sudo systemctl enable docker
            sudo docker run -d --name zookeeper-server --privileged=true -p 2181:2181 -e ALLOW_ANONYMOUS_LOGIN=yes bitnami/zookeeper:3.8.1
        EOF       
    }
    
    output "zookeeper_server" {
      value = "${module.ecs_zookeeper_service.instance_access_ipv4}:2181"
    }

    output "admin_passwd" {
      value = var.admin_passwd == "" ? nonsensitive(local.admin_passwd) : local.admin_passwd
    }