Add yorkie-analytics Helm chart with Starrocks and Kafka

build/charts/yorkie-analytics/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

build/charts/yorkie-analytics/Chart.lock

@@ -0,0 +1,9 @@
+dependencies:
+- name: kube-starrocks
+  repository: https://starrocks.github.io/starrocks-kubernetes-operator
+  version: 1.9.10
+- name: kafka
+  repository: oci://registry-1.docker.io/bitnamicharts
+  version: 31.3.1
+digest: sha256:0f2023982286f9d21800f964d4e6ed142569a478f5a30aac2367e68c2c953bf5
+generated: "2025-02-16T04:31:13.325546+09:00"

build/charts/yorkie-analytics/Chart.yaml

@@ -0,0 +1,34 @@
+apiVersion: v2
+name: yorkie-analytics
+description: yorkie-analytics provides analytics system with kafka and starrocks for analyzing yorkie cluster
+type: application
+icon: https://raw.githubusercontent.com/yorkie-team/yorkie-team.github.io/main/public/favicon-512x512.png
+maintainers:
+  - name: hackerwins
+    email: [email protected]
+  - name: krapie
+    email: [email protected]
+
+sources:
+  - https://github.com/yorkie-team/yorkie
+version: 0.6.0
+appVersion: "0.6.0"
+kubeVersion: ">=1.24.0-0"
+
+keywords:
+  - yorkie
+  - cluster
+  - kubernetes
+  - kafka
+  - starrocks
+
+# Kube Kafka and Starrocks stack dependencies
+dependencies:
+  - name: kube-starrocks
+    version: 1.9.10
+    repository: https://starrocks.github.io/starrocks-kubernetes-operator
+    condition: yorkie-analytics.starrocks.enabled
+  - name: kafka
+    version: 31.3.1
+    repository: oci://registry-1.docker.io/bitnamicharts
+    condition: yorkie-analytics.kafka.enabled

build/charts/yorkie-analytics/templates/NOTES.txt

@@ -0,0 +1,9 @@
+--- Install Complete ---
+{{ .Release.Name }} successfully installed!
+
+For next steps, follow:
+  $ curl https://github.com/yorkie-team/yorkie/tree/main/charts/yorkie-analytics
+
+To learn more about the release, try:
+  $ helm status {{ .Release.Name }}
+  $ helm get all {{ .Release.Name }}

build/charts/yorkie-analytics/templates/kafka/configmap.yaml

@@ -0,0 +1,24 @@
+{{- if index .Values "yorkie-analytics" "kafka" "enabled" }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.name }}-init-kafka-topics-script
+  namespace: {{ .Values.namespace }}
+  labels:
+    app.kubernetes.io/component: kafka
+    app.kubernetes.io/part-of: yorkie-analytics
+data:
+  init-kafka-topics.sh: |
+    #!/bin/bash
+
+    KAFKA_HOST={{ .Values.name }}-kafka.{{ .Values.namespace }}.svc.cluster.local:9092
+
+    echo -e 'Waiting for Kafka to be ready...'
+    kafka-topics.sh --bootstrap-server $KAFKA_HOST --list
+
+    echo -e 'Creating kafka topics'
+    kafka-topics.sh --bootstrap-server $KAFKA_HOST --create --if-not-exists --topic {{ index .Values "yorkie-analytics" "kafka" "topic" "name" }}
+
+    echo -e 'Successfully created the following topics:'
+    kafka-topics.sh --bootstrap-server $KAFKA_HOST --list
+{{- end }}

build/charts/yorkie-analytics/templates/kafka/job.yaml

@@ -0,0 +1,34 @@
+{{- if index .Values "yorkie-analytics" "kafka" "enabled" }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Values.name }}-init-kafka-topics-job
+  namespace: {{ .Values.namespace }}
+  labels:
+    app.kubernetes.io/component: kafka
+    app.kubernetes.io/part-of: yorkie-analytics
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  template:
+    spec:
+      serviceAccountName: starrocks
+      restartPolicy: OnFailure
+      initContainers:
+        - name: wait-for-kafka
+          image: bitnami/kubectl
+          command: [ "kubectl", "wait", "pod/{{ $.Values.name }}-kafka-controller-0", "--for", "condition=Ready", "--namespace", "{{ .Values.namespace }}", "--timeout", "300s" ]
+      containers:
+        - name: init-kafka-topics
+          image: {{ index .Values "yorkie-analytics" "kafka" "image" "repository" }}:{{ index .Values "yorkie-analytics" "kafka" "image" "tag" }}
+          command: [ "/bin/bash", "/etc/config/init-kafka-topics.sh" ]
+          volumeMounts:
+          - name: init-kafka-topics-script
+            mountPath: /etc/config
+      volumes:
+      - name: init-kafka-topics-script
+        configMap:
+          name: {{ .Values.name }}-init-kafka-topics-script
+  backoffLimit: 10
+{{- end }}

build/charts/yorkie-analytics/templates/starrocks/configmap.yaml

@@ -0,0 +1,75 @@
+{{- if index .Values "yorkie-analytics" "starrocks" "enabled" }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.name }}-init-starrocks-database-script
+  namespace: {{ .Values.namespace }}
+  labels:
+    app.kubernetes.io/component: starrocks
+    app.kubernetes.io/part-of: yorkie-analytics
+data:
+  init-user-events-db.sql: |
+    CREATE DATABASE IF NOT EXISTS yorkie;
+
+    USE yorkie;
+
+    CREATE TABLE user_events (
+        user_id VARCHAR(64),
+        timestamp DATETIME, 
+        event_type VARCHAR(32),
+        project_id VARCHAR(64),
+        user_agent VARCHAR(32)
+    ) ENGINE = OLAP  
+    DUPLICATE KEY(user_id)  
+    DISTRIBUTED BY HASH(user_id) BUCKETS 10
+    PROPERTIES (  
+        "replication_num" = "1"  
+    );
+
+  init-routine-load.sql: |
+    CREATE ROUTINE LOAD yorkie.events ON user_events
+    PROPERTIES
+    (
+        "format" = "JSON",
+        "desired_concurrent_number"="1"
+    )
+    FROM KAFKA
+    (   
+        "kafka_broker_list" = "{{ index .Values "yorkie-analytics" "starrocks" "routine-load" "kafka-broker-list" }}",
+        "kafka_topic" = "{{ index .Values "yorkie-analytics" "starrocks" "routine-load" "kafka-topic" }}",
+        "property.group.id" = "{{ index .Values "yorkie-analytics" "starrocks" "routine-load" "property-group-id" }}"
+    );
+
+  init-starrocks-database.sh: |
+    #!/bin/bash
+
+    STARROCKS_MYSQL_HOST=kube-starrocks-fe-service.{{ .Values.namespace }}.svc.cluster.local
+
+    sleep 5s
+    echo -e 'Checking Starrocks status'
+    mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root -e 'show frontends\G' | grep 'Alive: true' || echo -e 'Frontend is not ready'
+    mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root -e 'show backends\G' | grep 'Alive: true' || echo -e 'Starrocks BE is not ready'
+
+
+    echo -e 'Creating Yorkie database, tables and routine load'
+    mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root < /etc/config/init-user-events-db.sql
+
+    echo -e 'Checking Yorkie database'
+    mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root -e 'show databases\G'
+    mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root -e 'show databases\G' | grep 'Database: yorkie' || echo -e 'Yorkie database not found'
+
+    echo -e 'Checking user_event table'
+    mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root -e 'show tables from yorkie\G'
+    mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root -e 'show tables from yorkie\G' | grep 'Tables_in_yorkie: user_events' || echo -e 'user_events table not found'
+
+
+    sleep 5s
+    echo -e 'Creating routine load'
+    mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root < /etc/config/init-routine-load.sql
+
+    sleep 10s
+    echo -e 'Checking event routine load'
+
+    mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root -e 'show routine load from yorkie\G'
+    mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root -e 'show routine load from yorkie\G' | grep -E "State: NEED_SCHEDULE|State: RUNNING" || echo -e 'Routine load is not running'
+{{- end }}

build/charts/yorkie-analytics/templates/starrocks/job.yaml

@@ -0,0 +1,42 @@
+{{- if index .Values "yorkie-analytics" "starrocks" "enabled" }}      
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Values.name }}-init-starrocks-database-job
+  namespace: {{ .Values.namespace }}
+  labels:
+    app.kubernetes.io/component: starrocks
+    app.kubernetes.io/part-of: yorkie-analytics
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  template:
+    spec:
+      serviceAccountName: starrocks
+      restartPolicy: OnFailure
+      initContainers:
+        - name: wait-for-starrocks-fe
+          image: bitnami/kubectl
+          command: [ "kubectl", "wait", "pod/kube-starrocks-fe-0", "--for", "condition=Ready", "--namespace", "{{ .Values.namespace }}", "--timeout", "300s" ]
+        - name: wait-for-starrocks-be
+          image: bitnami/kubectl
+          command: [ "kubectl", "wait", "pod/kube-starrocks-be-0", "--for", "condition=Ready", "--namespace", "{{ .Values.namespace }}", "--timeout", "300s" ]
+        {{- if index .Values "yorkie-analytics" "kafka" "enabled" }}
+        - name: wait-for-kafka
+          image: bitnami/kubectl
+          command: [ "kubectl", "wait", "pod/{{ $.Values.name }}-kafka-controller-0", "--for", "condition=Ready", "--namespace", "{{ .Values.namespace }}", "--timeout", "300s" ]
+        {{- end }}
+      containers:
+        - name: init-starrocks-database
+          image: {{ index .Values "yorkie-analytics" "starrocks" "image" "repository" }}:{{ index .Values "yorkie-analytics" "starrocks" "image" "tag" }}
+          command: [ "/bin/bash", "/etc/config/init-starrocks-database.sh" ]
+          volumeMounts:
+          - name: init-starrocks-database-script
+            mountPath: /etc/config
+      volumes:
+      - name: init-starrocks-database-script
+        configMap:
+          name: {{ .Values.name }}-init-starrocks-database-script
+  backoffLimit: 10
+{{- end }}

build/charts/yorkie-analytics/values.yaml

@@ -0,0 +1,65 @@
+name: yorkie-analytics
+namespace: analytics
+
+yorkie-analytics:
+  starrocks:
+    enabled: true
+    image:
+      repository: &starrocks-fe-repo starrocks/fe-ubuntu
+      tag: &starrocks-fe-tag "3.3-latest"
+    routine-load:
+      kafka-broker-list: yorkie-analytics-kafka.analytics.svc.cluster.local:9092
+      kafka-topic: user-events
+      property-group-id: user_events_group
+  kafka:
+    enabled: true
+    image:
+      repository: bitnami/kafka
+      tag: latest
+    topic:
+      name: user-events
+
+kube-starrocks:
+  starrocks:
+    starrocksFESpec:
+      replicas: 1
+      image:
+        repository: *starrocks-fe-repo
+        tag: *starrocks-fe-tag
+      resources:
+        requests:
+          cpu: 1
+          memory: 1Gi
+      storageSpec:
+        name: fe
+
+    starrocksBeSpec:
+      replicas: 1
+      resources:
+        requests:
+          cpu: 1
+          memory: 2Gi
+      storageSpec:
+        name: be
+        storageSize: 15Gi
+
+    starrocksFeProxySpec:
+      enabled: false
+      service:
+        type: LoadBalancer
+
+kafka:
+  kraft:
+    clusterId: yorkie-analytics
+  provisioning:
+    replicationFactor: 1
+    numPartitions: 1
+  listeners:
+    client:
+      protocol: PLAINTEXT
+    controller:
+      protocol: PLAINTEXT
+  serviceAccount:
+    create: true
+  rbac:
+    create: true

build/charts/yorkie-cluster/templates/yorkie/deployment.yaml

@@ -81,12 +81,18 @@ spec:
           "{{ .Values.yorkie.ports.profilingPort }}",
           "--backend-gateway-addr",
           "{{ .Values.yorkie.name }}-gateway.{{ .Values.yorkie.namespace }}.svc.cluster.local",
-          {{- if and .Values.yorkie.args.kafkaAddresses .Values.yorkie.args.kafkaTopic }}
+          {{- if .Values.yorkie.args.kafkaAddresses }}
           "--kafka-addresses",
           "{{ .Values.yorkie.args.kafkaAddresses }}",
+          {{- if .Values.yorkie.args.kafkaTopic }}
           "--kafka-topic",
           "{{ .Values.yorkie.args.kafkaTopic }}",
           {{- end }}
+          {{- if .Values.yorkie.args.kafkaWriteTimeout }}
+          "--kafka-write-timeout",
+          "{{ .Values.yorkie.args.kafkaWriteTimeout }}",
+          {{- end }}
+          {{- end }}
         ]
         ports:
           - containerPort: {{ .Values.yorkie.ports.rpcPort }}

build/docker/analytics/docker-compose.yml

@@ -116,32 +116,34 @@ services:
     entrypoint: ["/bin/sh", "-c"]
     command: |
       "
+      STARROCKS_MYSQL_HOST=starrocks-fe
+
+      sleep 5s
       echo -e 'Checking Starrocks status'
-      mysql -u root -h starrocks-fe -P 9030 -e 'show frontends\\G' | grep 'Alive: true' || echo -e 'Frontend is not ready'
-      mysql -u root -h starrocks-fe -P 9030 -e 'show backends\\G' | grep 'Alive: true' || echo -e 'Backend is not ready'
+      mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root -e 'show frontends\\G' | grep 'Alive: true' || echo -e 'Frontend is not ready'
+      mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root -e 'show backends\\G' | grep 'Alive: true' || echo -e 'Backend is not ready'
 
 
       echo -e 'Creating Yorkie database, tables and routine load'
-      mysql -P 9030 -h starrocks-fe -u root < /init-user-events-db.sql
+      mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root < /init-user-events-db.sql
 
       echo -e 'Checking Yorkie database'
-      mysql -P 9030 -h starrocks-fe -u root -e 'show databases\\G'
-      mysql -P 9030 -h starrocks-fe -u root -e 'show databases\\G' | grep 'Database: yorkie' || echo -e 'Yorkie database not found'
+      mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root -e 'show databases\\G'
+      mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root -e 'show databases\\G' | grep 'Database: yorkie' || echo -e 'Yorkie database not found'
 
       echo -e 'Checking user_event table'
-      mysql -P 9030 -h starrocks-fe -u root -e 'show tables from yorkie\\G'
-      mysql -P 9030 -h starrocks-fe -u root -e 'show tables from yorkie\\G' | grep 'Tables_in_yorkie: user_events' || echo -e 'user_events table not found'
+      mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root -e 'show tables from yorkie\\G'
+      mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root -e 'show tables from yorkie\\G' | grep 'Tables_in_yorkie: user_events' || echo -e 'user_events table not found'
 
 
       sleep 5s
-
       echo -e 'Creating routine load'
-      mysql -P 9030 -h starrocks-fe -u root < /init-routine-load.sql
+      mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root < /init-routine-load.sql
 
+      sleep 10s
       echo -e 'Checking event routine load'
-
-      mysql -P 9030 -h starrocks-fe -u root -e 'show routine load from yorkie\\G'
-      mysql -P 9030 -h starrocks-fe -u root -e 'show routine load from yorkie\\G' | grep -E "State: NEED_SCHEDULE|State: RUNNING" || echo -e 'Routine load is not running'
+      mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root -e 'show routine load from yorkie\\G'
+      mysql -h $STARROCKS_MYSQL_HOST -P 9030 -u root -e 'show routine load from yorkie\\G' | grep -E "State: NEED_SCHEDULE|State: RUNNING" || echo -e 'Routine load is not running'
       "
 
 volumes: