[#3553] Hotfix Airy installation with custom Kafka

VERSION

@@ -1 +1 @@
-0.47.1
+0.47.2

cli/pkg/providers/minikube/minikube.go

@@ -80,7 +80,7 @@ func (p *provider) startCluster(providerConfig map[string]string) error {
 	cpusArg := "--cpus=" + minikubeCpus
 	memoryArg := "--memory=" + minikubeMemory
 	args := []string{"start", "--extra-config=apiserver.service-node-port-range=1-65535", "--ports=80:80", driverArg, runtimeArg, cpusArg, memoryArg}
-	// Prevent minikube download progress bar from polluting the output
+	// Prevent minikube download progress bar from polluting the output by downloading silently first
 	_, err := runGetOutput(append(args, "--download-only")...)
 	if err != nil {
 		return fmt.Errorf("downloading minikube files err: %v", err)

docs/docs/changelog.md

@@ -3,6 +3,20 @@ title: Changelog
 sidebar_label: 📝 Changelog
 ---
 
+## 0.47.2
+
+#### Hotfix
+
+* [[#3553](https://github.com/airyhq/airy/issues/3553)] Fix using a custom Kafka when installing Airy via Helm
+
+#### Airy CLI
+
+You can download the Airy CLI for your operating system from the following links:
+
+[MacOS](https://airy-core-binaries.s3.amazonaws.com/0.47.2/darwin/amd64/airy)
+[Linux](https://airy-core-binaries.s3.amazonaws.com/0.47.2/linux/amd64/airy)
+[Windows](https://airy-core-binaries.s3.amazonaws.com/0.47.2/windows/amd64/airy.exe)
+
 ## 0.47.1
 
 #### Hotfix
@@ -1233,6 +1247,3 @@ You can download the Airy CLI for your operating system from the following links
 ## Hotfix 0.26.3
 
 [[#2192](https://github.com/airyhq/airy/issues/2192)] Inbox crashing when selecting conversations in filtered view [[#2193](https://github.com/airyhq/airy/pull/2193)]
-## Hotfix 0.26.2
-
-[[#2187](https://github.com/airyhq/airy/issues/2187)] Hotfix chat plugin async bundle loading failed on installed websites

docs/docs/getting-started/installation/helm.md

@@ -106,7 +106,7 @@ For more information refer to the [official DigitalOcean Guide](https://docs.dig
 `Airy Core` can be created on Minikube with predefined settings, using the [Airy CLI](/cli/introduction). However, if you want to create your custom Minikube instance, for example with custom settings for CPU and RAM, you can also do that with the [Minikube](https://minikube.sigs.k8s.io/docs/start/) utility:
 
 ```sh
-minikube -p airy start --driver=virtualbox --cpus=4 --memory=7168 --extra-config=apiserver.service-nodeport-range=1-65535
+minikube start -p airy--driver=docker --cpus=4 --memory=7168 --ports=80:80 --extra-config=apiserver.service-node-port-range=1-65535
 ```
 
 The `apiserver.service-nodeport-range` settings is needed if you want to use port 80 on the Minikube VM as the NodePort for the ingress controller service.

docs/sidebars.js

@@ -118,6 +118,7 @@ module.exports = {
         'guides/monitoring',
         'guides/backup',
         'guides/component-reset',
+        'guides/remote-kafka-cluster',
       ],
     },
     'changelog',

infrastructure/helm-chart/templates/config/kafka.yaml

@@ -4,6 +4,12 @@ metadata:
   name: kafka-config
 data:
   KAFKA_BROKERS: {{ .Values.config.kafka.brokers }}
+{{- if .Values.config.kafka.authJaas }}
+  AUTH_JAAS: {{ .Values.config.kafka.authJaas }}
+{{- end }}
+  KAFKA_MINIMUM_REPLICAS: "{{ .Values.config.kafka.minimumReplicas }}"
+{{- if .Values.config.kafka.zookeepers }}
   ZOOKEEPER: {{ .Values.config.kafka.zookeepers }}
+{{- end }}
   KAFKA_SCHEMA_REGISTRY_URL: {{ .Values.config.kafka.schemaRegistryUrl }}
   KAFKA_COMMIT_INTERVAL_MS: "{{ .Values.config.kafka.commitInterval }}"

infrastructure/helm-chart/templates/provisioning/kafka-create-topics.yaml

@@ -30,6 +30,19 @@ data:
       echo "Using ${AIRY_CORE_NAMESPACE} to namespace topics"
     fi
 
+    if [ -n "${AUTH_JAAS}" ]; then
+      cat <<EOF > /opt/kafka/jaas.config
+    security.protocol=SASL_SSL
+    sasl.jaas.config=$AUTH_JAAS
+    sasl.mechanism=PLAIN
+    EOF
+      CONNECTION_OPTS+=(--command-config /opt/kafka/jaas.config)
+      echo "Using jaas authentication for connecting to Kafka"
+    fi
+
+    echo "Creating Kafka topics"
+
+
 
     kafka-topics.sh --create --if-not-exists --zookeeper "${ZOOKEEPER}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.channels" 
 

infrastructure/helm-chart/values.yaml

@@ -12,12 +12,14 @@ config:
   kafka:
     brokers: "kafka-headless:9092"
     zookeepers: "zookeeper:2181"
+    authJaas: ""
+    minimumReplicas: 1
     schemaRegistryUrl: "http://schema-registry:8081"
     commitInterval: 1000
 provisioning:
   kafka:
     image: ghcr.io/airyhq/infrastructure/kafka
-    imageTag: 2.7.0
+    imageTag: 3.2.0
     namespaceTopics: false
 tools:
   segment:

infrastructure/tools/topics/src/main/java/co/airy/tools/topics/Application.java

@@ -53,7 +53,20 @@ public static void main(String[] args) {
                 "    then\n" +
                 "      AIRY_CORE_NAMESPACE=\"${AIRY_CORE_NAMESPACE}.\"\n" +
                 "      echo \"Using ${AIRY_CORE_NAMESPACE} to namespace topics\"\n" +
-                "    fi";
+                "    fi\n" +
+                "\n" +
+                "    if [ -n \"${AUTH_JAAS}\" ]; then\n" +
+                "      cat <<EOF > /opt/kafka/jaas.config\n" +
+                "    security.protocol=SASL_SSL\n" +
+                "    sasl.jaas.config=$AUTH_JAAS\n" +
+                "    sasl.mechanism=PLAIN\n" +
+                "    EOF\n" +
+                "      CONNECTION_OPTS+=(--command-config /opt/kafka/jaas.config)\n" +
+                "      echo \"Using jaas authentication for connecting to Kafka\"\n" +
+                "    fi\n" +
+                "\n" +
+                "    echo \"Creating Kafka topics\"\n"
+                ;
 
         TopicsFinder finder = new TopicsFinder();
 

lib/go/config/airy_yaml.go

@@ -31,6 +31,7 @@ type KafkaConf struct {
 	Zookeeper         string `yaml:"zookeeper" json:"zookeeper"`
 	SchemaRegistryUrl string `yaml:"schemaRegistryUrl" json:"schemaRegistryUrl"`
 	AuthJaas          string `yaml:"authJaas" json:"authJaas"`
+	MinimumReplicas   string `yaml:"minimumReplicas" json:"minimumReplicas"`
 }
 
 type ComponentsConf map[string]map[string]string

lib/go/config/kafka.go

@@ -23,5 +23,9 @@ func GetKafkaData(s KafkaConf) map[string]string {
 		m["AUTH_JAAS"] = s.AuthJaas
 	}
 
+	if s.MinimumReplicas != "" {
+		m["KAFKA_MINIMUM_REPLICAS"] = s.MinimumReplicas
+	}
+
 	return m
 }

lib/java/spring/kafka/streams/src/main/resources/kafka-streams.properties

@@ -1,6 +1,6 @@
 kafka.brokers=${KAFKA_BROKERS}
 kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL}
 kafka.commit-interval-ms=${KAFKA_COMMIT_INTERVAL_MS:1000}
-
+kafka.replication.factor=${KAFKA_MINIMUM_REPLICAS:1}
 # Only used for testing:
 kafka.cleanup=${KAFKA_CLEANUP:false}