mtls

README.md

@@ -14,6 +14,10 @@ This repository contains examples of use cases that utilize Decodable streaming
 | [Apache Pinot](pinot) | Transforming osquery logs to Apache Pinot and Superset |
 | [Apache Druid](druid) | This example sends covid 19 data to Decodable using it's REST API. The data is then cleansed using Decodable SQL and send the data to a Kafka sink.  |
 | [Rockset](rockset) | We will be utilizing a cloud MQTT broker and AWS Kinesis to capture and stream data. Decodable will be responsible for preparing and aggregating the data prior to reaching the real-time analytical database (Rockset) |
+| [Tinybird](tinybird) | We write data to Tinybird and build a simple real time web application. |
+| [Apache Kafka](kafka2s3) | Installing Apache Kafka on EC2 and writing to S3 with Decodable |
+| [Apache Kafka mTLS](mtls) | We install Apache Kafka on EC2 and configure it with mTLS and configure Decodable to read from it |
+
 
 
 

kafka2s3/mtls/Makefile → mtls/Makefile

@@ -97,7 +97,8 @@ send.server:
 	scp -i ${PEMDEVW} ssl/kafka.server.*.jks ec2-user@$(BROKER_HOST):kafka_2.13-3.2.1/ssl/
 
 ssl.verify:
-	openssl s_client -connect ${BROKER_HOST}:9093
+	# openssl s_client -connect ${BROKER_HOST}:9093
+	openssl s_client -connect 34.222.216.72:9093
 
 delete.stream:
 	-@decodable stream delete $(shell decodable stream list -o json | jq -sr '.[] |select(.name=="crypto_mtls")|.id ' )
@@ -151,7 +152,7 @@ create.conn.mtls:
 		--prop properties.ssl.endpoint.identification.algorithm=\
 		--prop tls.broker.certificate=@ssl/server-cert-signed \
 
-recrecate: mtls deactivate.conn.mtls delete.conn.mtls delete.stream create.stream create.conn.mtls add.decodable
+recreate: mtls deactivate.conn.mtls delete.conn.mtls delete.stream create.stream create.conn.mtls add.decodable
 
 add.decodable:
 	decodable connection \
@@ -174,6 +175,7 @@ java.consume:
 		--consumer.config client.properties
 
 java.produce:
+	KAFKA_OPTS="-Djavax.net.debug=ssl"
 	~/development/cp/confluent-7.1.2/bin/kafka-console-producer \
 		--bootstrap-server $(BROKER_HOST):9093 \
 		--topic crytpo_mtls \

kafka2s3/mtls/README.md → mtls/README.md

@@ -48,6 +48,11 @@ classDef file fill:green,stroke:#333,stroke-width:4px
 ## server.properties file
 Requires Kafka restart.
 
+set this environment property to show SSL debug logs.
+```bash
+export KAFKA_OPTS=-Djavax.net.debug=all
+```
+
 ```properties
 listeners=PLAINTEXT://0.0.0.0:9092,SSL://0.0.0.0:9093
 advertised.listeners=PLAINTEXT://<HOSTNAME>:9092,SSL://<HOSTNAME>:9093
@@ -58,6 +63,8 @@ ssl.key.password=yourpassword
 ssl.truststore.location=<path>/kafka.server.truststore.jks
 ssl.truststore.password=yourpassword
 
+ssl.client.auth=required
+
 ```
 
 Verify the SSL port
@@ -95,6 +102,23 @@ Consumer
 ./kafka-console-consumer.sh --broker-list HOSTNAME:9093 --topic mytopic --consumer.config PATH_TO_THE_ABOVE_PROPERTIES
 ```
 
+# Configuring Decodable
+
+The common name (CN) must match exactly the fully qualified domain name (FQDN) of the server. The client compares the CN with the DNS domain name to ensure that it is indeed connecting to the desired server, not a malicious one. The hostname of the server can also be specified in the Subject Alternative Name (SAN). Since the distinguished name is used as the server principal when SSL is used as the inter-broker security protocol, it is useful to have hostname as a SAN rather than the CN.
+
+To show the CN or SAN in a signed certificate, run the command below:
+
+```bash
+openssl x509 -noout -subject -in your-signed-cert
+```
+
+Host name verification of servers is enabled by default for client connections as well as inter-broker connections to prevent man-in-the-middle attacks. Server host name verification may be disabled by setting ssl.endpoint.identification.algorithm to an empty string. For example,
+
+```properties
+ssl.endpoint.identification.algorithm=
+```
+
+
 # mTLS
 Client authentication. Broker work is same as above.