OpenstackTestLab-Kafka.yml

# The version of the Xpanse description language
version: 1.0
# The category of the service.
category: middleware
# The Service provided by the ISV, the name will be shown on the console as a service.
name: kafka-cluster
# The version of the service, the end-user can select the version they want to deploy.
serviceVersion: 1.0.0
# For the users may have more than one service, the @namespace can be used to separate the clusters.
description: This is an enhanced Kafka cluster services by ISV-A.
namespace: ISV-A
# Icon for the service.
icon: |
  data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAACRAQMAAAAPc4+9AAAAAXNSR0IB2cksfwAAAAlwSFlzAAALEwAACxMBAJqcGAAAAAZQTFRF+/v7Hh8gVD0A0wAAAcVJREFUeJzNlc1twzAMhSX44KNH0CgaTd6gK3kUd4McDVTwq/hjiUyaIkV7qNA2/QCFIh+ppxB+svLNEqqBGTC0ANugBOwmCGDCFOAwIWGDOoqoODtN2BdL6wxD9NMTO9tXPa1PqL5M30W5p8lm5vNcF0t7ahSrVguqNqmMokRW4YQucVjBCBWH1Z2g3WDlW2skoYU+2x8JOtGedBF3k2iXMO0j16iUiI6gxzPdQhnU/s2G9pCO57QY2r6hvjPbKJHq7DRTRXT60avtuTRdbrFJI3mSZhNOqYjVbd99YyK1QKWzEqSWrE0k07U60uPaelflMzaaeu1KBuurHSsn572I1KWy2joX5ZBfWbS/VEt50H5P6aL4JxTuyJ/+QCNPX4PWF3Q8Xe1eF9FsLdD2VaOnaP2hWvs+zI58/7i3vH3nRFtDZpyTUNaZkON5XnBNsp8lrmDMrpvBr+b6pUl+4XbkQdndqnzYGzfuJm1JmIWimIbe6dndd/bk7gVce/cJdo3uIeLJl7+I2xTnPek67mjtDeppE7b03Ov+kSfDe3JweW53njxeGfXkaz28VeYd86+af/H8a7hgJKaebILaFzakLfxyfQLTxVB6K1K9KQAAAABJRU5ErkJggg==
# Reserved for CSP: HuaweiCloud,FlexibleEngine,OpenstackTestlab,PlusServer,RegioCloud and ...
cloudServiceProvider:
  name: OpenstackTestlab
  regions:
    - name: RegionOne
      area: Western Europe
    - name: RegionTwo
      area: Western Europe
billing:
  # The supported mode of billing (`Fixed`, `Pay per Use`)
  billingModes:
    - Fixed
serviceHostingType: self
# The flavor of the service, the @category/@name/@version/@flavor can locate the specific service to be deployed.
flavors:
  serviceFlavors:
    - name: 1-zookeeper-with-3-worker-nodes-normal
      priority: 2
      # The pricing of the flavor.
      pricing:
        # Used to calculate charges when users do not select 'pay_per_use' as the billing mode.
        fixedPrice:
          cost: 95.00
          currency: USD
          period: monthly
      # Properties for the service, which can be used by the deployment.
      properties:
        worker_nodes_count: 3
        flavor_name: cirros256
      features:
        - High Availability
        - Maximum performance
    - name: 1-zookeeper-with-3-worker-nodes-performance
      priority: 1
      # The pricing of the flavor.
      pricing:
        # Used to calculate charges when users do not select 'pay_per_use' as the billing mode.
        fixedPrice:
          cost: 122.0
          currency: USD
          period: monthly
      # Properties for the service, which can be used by the deployment.
      properties:
        worker_nodes_count: 3
        flavor_name: cirros512
      features:
        - High Availability
        - Maximum performance
  modificationImpact:
    isDataLost: true
    isServiceInterrupted: true
  isDowngradeAllowed: true
# The contact details of the service.
serviceProviderContactDetails:
  emails: [ "test@test.com" ]
configurationParameters:
  - name: kafka_cfg_message_max_bytes
    kind: variable
    dataType: number
    description: Parameters used to configure the Kafka server or client to set the maximum number of bytes for Kafka messages.
    initialValue: 1048576
    modificationImpact:
      isDataLost: true
      isServiceInterrupted: true
    isReadOnly: false
  - name: kafka_cfg_log_dirs
    kind: variable
    dataType: string
    description: Parameters for the storage location of Kafka log data
    initialValue: /var/lib/kafka/logs
    modificationImpact:
      isDataLost: true
      isServiceInterrupted: true
    isReadOnly: false
  - name: kafka_cfg_num_io_threads
    kind: variable
    dataType: number
    description: Parameter used to set the number of I/O threads to handle kafka network requests
    initialValue: 8
    modificationImpact:
      isDataLost: true
      isServiceInterrupted: true
    isReadOnly: false
  - name: kafka_log_flush_interval_messages
    kind: variable
    dataType: number
    description: Kafka's log flush strategy, specifies the parameters that trigger a log flush operation after how many messages are written.
    initialValue: 10000
    modificationImpact:
      isDataLost: true
      isServiceInterrupted: true
    isReadOnly: false
  - name: kafka_offsets_topic_replication_factor
    kind: variable
    dataType: number
    description: Replication factor configuration parameters for the dedicated topic (offsets topic) used in the Kafka cluster to consume offset information for the consumer storage group
    initialValue: 3
    modificationImpact:
      isDataLost: true
      isServiceInterrupted: true
    isReadOnly: false
deployment:
  # kind, Supported values are terraform, opentofu.
  kind: terraform
  serviceAvailabilityConfigs:
    - displayName: Availability Zone
      varName: availability_zone
      mandatory: false
      description: The availability zone to deploy the service instance. If the value is empty, the service instance will be deployed in a random availability zone.
  # Context for deployment: the context including some kind of parameters for the deployment, such as fix_env, fix_variable, env, variable, env_env, env_variable.
  # - fix_env: Values for variable of this type are defined by the managed service provider in the OCL template. Runtime will inject it to deployer as environment variables. This variable is not visible to the end user.
  # - fix_variable: Values for variable of this type are defined by the managed service provider in the OCL template. Runtime will inject it to deployer as usual variables. This variable is not visible to the end user.
  # - env: Value for a variable of this type can be provided by end user. If marked as mandatory then end user must provide value to this variable. If marked as optional and if end user does not provided it, then the fallback value to this variable is read by runtime (it can read from other sources, e.g., OS env variables). This variable is injected as a environment variable to the deployer.
  # - variable: Value for a variable of this type can be provided by end user. . If marked as mandatory then end user must provide value to this variable. If marked as optional and if end user does not provided it, then the fallback value to this variable is read by runtime (it can read from other sources, e.g., OS env variables). This variable is injected as a regular variable to the deployer.
  # - env_env: Value to this variable is read by runtime (it can read from other sources, e.g., OS env variables) and injected as a environment variable to the deployer. End user cannot see or change this variable.
  # - env_variable: Value to this variable is read by runtime (it can read from other sources, e.g., OS env variables) and injected as a regular variable to the deployer. End user cannot see or change this variable.
  # The parameters will be used to generate the API of the managed service.
  variables:
    - name: OS_AUTH_URL
      description: Openstack cloud instance to be used.
      kind: fix_env
      dataType: string
      mandatory: true
      valueSchema:
        minLength: 1
        maxLength: 256
      sensitiveScope: none
      # this is an example test instance of openstack
      value: "http://119.8.215.244/identity/v3"
      modificationImpact:
        isDataLost: true
        isServiceInterrupted: true
    - name: admin_passwd
      description: The admin password of all nodes in the Kafka cluster instance. If the value is empty, will create a random password.
      kind: variable
      dataType: string
      mandatory: false
      valueSchema:
        minLength: 8
        maxLength: 16
        pattern: ^(?=.*?[A-Z])(?=.*?[a-z])(?=.*?[0-9])(?=.*?[#?!@$%^&*-]).{8,16}$
      modificationImpact:
        isDataLost: true
        isServiceInterrupted: true
    - name: vpc_name
      description: The vpc name of all nodes in the Kafka cluster instance. If the value is empty, will use the default value to find or create VPC.
      kind: variable
      dataType: string
      example: "kafka-vpc-default"
      mandatory: false
      value: "kafka-vpc-default"
      modificationImpact:
        isDataLost: true
        isServiceInterrupted: true
    - name: subnet_name
      description: The sub network name of all nodes in the Kafka cluster instance. If the value is empty, will use the default value to find or create subnet.
      kind: variable
      dataType: string
      example: "kafka-subnet-default"
      mandatory: false
      value: "kafka-subnet-default"
      modificationImpact:
        isDataLost: true
        isServiceInterrupted: true
    - name: secgroup_name
      description: The security group name of all nodes in the Kafka cluster instance. If the value is empty, will use the default value to find or create security group.
      kind: variable
      dataType: string
      example: "kafka-secgroup-default"
      mandatory: false
      value: "kafka-secgroup-default"
      modificationImpact:
        isDataLost: true
        isServiceInterrupted: true
  deployer: |
    variable "region" {
      type        = string
      description = "The region to deploy the Kafka cluster instance."
    }
    
    variable "availability_zone" {
      type        = string
      default     = ""
      description = "The availability zone to deploy the Kafka cluster instance."
    }
    
    variable "flavor_name" {
      type        = string
      default     = "cirros256"
      description = "The flavor_name of all nodes in the Kafka cluster instance."
    }

    variable "worker_nodes_count" {
      type        = string
      default     = 3
      description = "The worker nodes count in the Kafka cluster instance."
    }

    variable "admin_passwd" {
      type= string
      default = ""
      description = "The root password of all nodes in the Kafka cluster instance."
    }

    variable "vpc_name" {
      type        = string
      default     = "kafka-vpc-default"
      description = "The vpc name of all nodes in the Kafka cluster instance."
    }

    variable "subnet_name" {
      type        = string
      default     = "kafka-subnet-default"
      description = "The subnet name of all nodes in the Kafka cluster instance."
    }

    variable "secgroup_name" {
      type        = string
      default     = "kafka-secgroup-default"
      description = "The security group name of all nodes in the Kafka cluster instance."
    }
    
    terraform {
      required_providers {
        openstack = {
          source  = "terraform-provider-openstack/openstack"
          version = "~> 1.53.0"
        }
      }
    }
    
    provider "openstack" {
      region = var.region
    }
    
    data "openstack_compute_availability_zones_v2" "osc-az" {}

    data "openstack_networking_network_v2" "existing" {
      name  = var.vpc_name
      count = length(data.openstack_networking_network_v2.existing)
    }

    data "openstack_networking_subnet_v2" "existing" {
      name  = var.subnet_name
      count = length(data.openstack_networking_subnet_v2.existing)
    }

    data "openstack_networking_secgroup_v2" "existing" {
      name  = var.secgroup_name
      count = length(data.openstack_networking_secgroup_v2.existing)
    }

    locals {
      availability_zone = var.availability_zone == "" ? data.openstack_compute_availability_zones_v2.osc-az.names[0] : var.availability_zone
      admin_passwd      = var.admin_passwd == "" ? random_password.password.result : var.admin_passwd
      vpc_id            = length(data.openstack_networking_network_v2.existing) > 0 ? data.openstack_networking_network_v2.existing[0].id : openstack_networking_network_v2.new[0].id
      subnet_id         = length(data.openstack_networking_subnet_v2.existing) > 0 ? data.openstack_networking_subnet_v2.existing[0].id : openstack_networking_subnet_v2.new[0].id
      secgroup_id       = length(data.openstack_networking_secgroup_v2.existing) > 0 ? data.openstack_networking_secgroup_v2.existing[0].id : openstack_networking_secgroup_v2.new[0].id
      secgroup_name     = length(data.openstack_networking_secgroup_v2.existing) > 0 ? data.openstack_networking_secgroup_v2.existing[0].name : openstack_networking_secgroup_v2.new[0].name
    }

    resource "openstack_networking_network_v2" "new" {
      count = length(data.openstack_networking_network_v2.existing) == 0 ? 1 : 0
      name  = "${var.vpc_name}-${random_id.new.hex}"
    }

    resource "openstack_networking_subnet_v2" "new" {
      count      = length(data.openstack_networking_subnet_v2.existing) == 0 ? 1 : 0
      network_id     = local.vpc_id
      name       = "${var.subnet_name}-${random_id.new.hex}"
      cidr       = "192.168.10.0/24"
      gateway_ip = "192.168.10.1"
    }

    resource "openstack_networking_secgroup_v2" "new" {
      count       = length(data.openstack_networking_secgroup_v2.existing) == 0 ? 1 : 0
      name        = "${var.secgroup_name}-${random_id.new.hex}"
      description = "Kafka cluster security group"
    }

    resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_0" {
      count             = length(data.openstack_networking_secgroup_v2.existing) == 0 ? 1 : 0
      direction         = "ingress"
      ethertype         = "IPv4"
      protocol          = "tcp"
      port_range_min    = 22
      port_range_max    = 22
      remote_ip_prefix  = "121.37.117.211/32"
      security_group_id = local.secgroup_id
    }

    resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_1" {
      count             = length(data.openstack_networking_secgroup_v2.existing) == 0 ? 1 : 0
      direction         = "ingress"
      ethertype         = "IPv4"
      protocol          = "tcp"
      port_range_min    = 2181
      port_range_max    = 2181
      remote_ip_prefix  = "121.37.117.211/32"
      security_group_id = local.secgroup_id
    }

    resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_2" {
      count             = length(data.openstack_networking_secgroup_v2.existing) == 0 ? 1 : 0
      direction         = "ingress"
      ethertype         = "IPv4"
      protocol          = "tcp"
      port_range_min    = 9092
      port_range_max    = 9093
      remote_ip_prefix  = "121.37.117.211/32"
      security_group_id = local.secgroup_id
    }

    resource "random_id" "new" {
      byte_length = 4
    }

    resource "random_password" "password" {
      length           = 12
      upper            = true
      lower            = true
      numeric          = true
      special          = true
      min_special      = 1
      override_special = "#%@"
    }

    resource "openstack_compute_keypair_v2" "keypair" {
      name = "keypair-k8s-${random_id.new.hex}"
    }

    data "openstack_images_image_v2" "image" {
      name        = "cirros-0.5.2-x86_64-disk"
      most_recent = true
    }

    resource "openstack_compute_instance_v2" "zookeeper" {
      availability_zone  = local.availability_zone
      name               = "kafka-zookeeper-${random_id.new.hex}"
      flavor_name        = var.flavor_name
      security_groups    = [ local.secgroup_name ]
      image_id           = data.openstack_images_image_v2.image.id
      key_pair           = openstack_compute_keypair_v2.keypair.name
      network {
        uuid = local.vpc_id
      }
      user_data = <<EOF
        #!bin/bash
        echo root:${local.admin_passwd} | sudo chpasswd
        sudo systemctl start docker
        sudo systemctl enable docker
        sudo docker run -d --name zookeeper-server --privileged=true -p 2181:2181 -e ALLOW_ANONYMOUS_LOGIN=yes bitnami/zookeeper:3.8.1
      EOF
    }

    resource "openstack_compute_instance_v2" "kafka-broker" {
      count              = var.worker_nodes_count
      availability_zone  = local.availability_zone
      name               = "kafka-broker-${count.index}-${random_id.new.hex}"
      flavor_name        = var.flavor_name
      security_groups    = [ local.secgroup_name ]
      image_id           = data.openstack_images_image_v2.image.id
      key_pair           = openstack_compute_keypair_v2.keypair.name
      network {
        uuid = local.vpc_id
      }
      user_data = <<EOF
        #!bin/bash
        echo root:${local.admin_passwd} | sudo chpasswd
        sudo systemctl start docker
        sudo systemctl enable docker
        private_ip=$(ifconfig | grep -A1 "eth0" | grep 'inet' | awk -F ' ' ' {print $2}'|awk ' {print $1}')
        sudo docker run -d --name kafka-server --restart always -p 9092:9092 -p 9093:9093  -e KAFKA_BROKER_ID=${count.index}  -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://$private_ip:9092 -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 -e ALLOW_PLAINTEXT_LISTENER=yes -e KAFKA_CFG_ZOOKEEPER_CONNECT=${openstack_compute_instance_v2.zookeeper.access_ip_v4}:2181 bitnami/kafka:3.3.2
      EOF
      depends_on = [
        openstack_compute_instance_v2.zookeeper
      ]
    }

    output "zookeeper_server" {
      value = "${openstack_compute_instance_v2.zookeeper.access_ip_v4}:2181"
    }

    output "admin_passwd" {
      value = var.admin_passwd == "" ? nonsensitive(local.admin_passwd) : local.admin_passwd
    }