-
Notifications
You must be signed in to change notification settings - Fork 208
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
added mTLS example but not working due to hostname verification
- Loading branch information
Showing
4 changed files
with
235 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
--- | ||
services: | ||
|
||
mosquitto: | ||
image: eclipse-mosquitto:1.6.13 | ||
hostname: mosquitto | ||
container_name: mosquitto | ||
ports: | ||
- 9001:9001 | ||
- 1883:1883 | ||
volumes: | ||
- ../../ccloud/fm-mqtt-source/mtls/mosquitto.conf:/mosquitto/config/mosquitto.conf | ||
- ../../ccloud/fm-mqtt-source/password:/etc/mosquitto/passwd | ||
- ../../ccloud/fm-mqtt-source/security/snakeoil-ca-1.crt:/tmp/ca.crt | ||
- ../../ccloud/fm-mqtt-source/security/mosquitto.certificate.pem:/tmp/server.crt | ||
- ../../ccloud/fm-mqtt-source/security/mosquitto.key:/tmp/server.key | ||
|
||
# https://ngrok.com/docs/using-ngrok-with/docker/ | ||
ngrok: | ||
image: ngrok/ngrok:latest | ||
hostname: ngrok | ||
container_name: ngrok | ||
ports: | ||
- 4040:4040 | ||
restart: unless-stopped | ||
links: | ||
- mosquitto | ||
command: | ||
- "start" | ||
- "--all" | ||
- "--log=stdout" | ||
- "--config" | ||
- "/etc/ngrok.yml" | ||
volumes: | ||
- ../../ccloud/fm-mqtt-source/ngrok.yml:/etc/ngrok.yml | ||
environment: | ||
NGROK_AUTHTOKEN: $NGROK_AUTH_TOKEN |
108 changes: 108 additions & 0 deletions
108
ccloud/fm-mqtt-source/fully-managed-mqtt-source-mtls.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,108 @@ | ||
#!/bin/bash | ||
set -e | ||
|
||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" | ||
source ${DIR}/../../scripts/utils.sh | ||
|
||
cd ${DIR}/security | ||
log "🔐 Generate keys and certificates used for SSL" | ||
docker run -u0 --rm -v $PWD:/tmp ${CP_CONNECT_IMAGE}:${CONNECT_TAG} bash -c "/tmp/certs-create.sh > /dev/null 2>&1 && chown -R $(id -u $USER):$(id -g $USER) /tmp/ && chmod a+r /tmp/*" | ||
base64_truststore=$(cat $PWD/kafka.connect.truststore.jks | base64 | tr -d '\n') | ||
base64_keystore=$(cat $PWD/kafka.connect.keystore.jks | base64 | tr -d '\n') | ||
cd ${DIR} | ||
|
||
NGROK_AUTH_TOKEN=${NGROK_AUTH_TOKEN:-$1} | ||
|
||
display_ngrok_warning | ||
|
||
bootstrap_ccloud_environment | ||
|
||
|
||
|
||
set +e | ||
playground topic delete --topic mqtt-source-1 | ||
set -e | ||
|
||
docker compose -f docker-compose.mtls.yml build | ||
docker compose -f docker-compose.mtls.yml down -v --remove-orphans | ||
docker compose -f docker-compose.mtls.yml up -d --quiet-pull | ||
|
||
sleep 5 | ||
|
||
log "Waiting for ngrok to start" | ||
while true | ||
do | ||
container_id=$(docker ps -q -f name=ngrok) | ||
if [ -n "$container_id" ] | ||
then | ||
status=$(docker inspect --format '{{.State.Status}}' $container_id) | ||
if [ "$status" = "running" ] | ||
then | ||
log "Getting ngrok hostname and port" | ||
NGROK_URL=$(curl --silent http://127.0.0.1:4040/api/tunnels | jq -r '.tunnels[0].public_url') | ||
NGROK_HOSTNAME=$(echo $NGROK_URL | cut -d "/" -f3 | cut -d ":" -f 1) | ||
NGROK_PORT=$(echo $NGROK_URL | cut -d "/" -f3 | cut -d ":" -f 2) | ||
|
||
if ! [[ $NGROK_PORT =~ ^[0-9]+$ ]] | ||
then | ||
log "NGROK_PORT is not a valid number, keep retrying..." | ||
continue | ||
else | ||
break | ||
fi | ||
fi | ||
fi | ||
log "Waiting for container ngrok to start..." | ||
sleep 5 | ||
done | ||
|
||
connector_name="MqttSourceMTLS_$USER" | ||
set +e | ||
playground connector delete --connector $connector_name > /dev/null 2>&1 | ||
set -e | ||
|
||
log "Creating fully managed connector" | ||
playground connector create-or-update --connector $connector_name << EOF | ||
{ | ||
"connector.class": "MqttSource", | ||
"name": "$connector_name", | ||
"kafka.auth.mode": "KAFKA_API_KEY", | ||
"kafka.api.key": "$CLOUD_KEY", | ||
"kafka.api.secret": "$CLOUD_SECRET", | ||
"output.data.format": "AVRO", | ||
"kafka.topic": "mqtt-source-1", | ||
"mqtt.qos": "2", | ||
"mqtt.server.uri" : "ssl://$NGROK_HOSTNAME:$NGROK_PORT", | ||
"mqtt.topics":"my-mqtt-topic", | ||
"mqtt.username": "myuser", | ||
"mqtt.password": "mypassword", | ||
"mqtt.ssl.trust.store.file": "data:text/plain;base64,$base64_truststore", | ||
"mqtt.ssl.trust.store.password": "confluent", | ||
"mqtt.ssl.key.store.file": "data:text/plain;base64,$base64_keystore", | ||
"mqtt.ssl.key.store.password": "confluent", | ||
"mqtt.ssl.key.password": "confluent", | ||
"tasks.max" : "1" | ||
} | ||
EOF | ||
wait_for_ccloud_connector_up $connector_name 180 | ||
|
||
|
||
# MqttSourceMTLS_vsaboulin ❌ FAILED 🤔 N/A connector: Unable to validate configuration. If an update was made to the configuration, this means that the configuration was invalid, and the connector continues to operate on a previous configuration that passed validation. Errors: | ||
# mqtt.server.uri: MqttException (0) - javax.net.ssl.SSLHandshakeException: No subject alternative DNS name matching 4.tcp.eu.ngrok.io found. validation_errors: mqtt.server.uri: MqttException (0) - javax.net.ssl.SSLHandshakeException: No subject alternative DNS name matching 4.tcp.eu.ngrok.io found. | ||
|
||
|
||
sleep 5 | ||
|
||
log "Send message to MQTT in my-mqtt-topic topic" | ||
docker exec mosquitto sh -c 'mosquitto_pub -h localhost -p 8883 -u "myuser" -P "mypassword" -t "my-mqtt-topic" -m "sample-msg-1" --cafile /tmp/ca.crt --key /tmp/server.key --cert /tmp/server.crt' | ||
|
||
sleep 5 | ||
|
||
log "Verify we have received the data in mqtt-source-1 topic" | ||
playground topic consume --topic mqtt-source-1 --min-expected-messages 1 --timeout 60 | ||
|
||
log "Do you want to delete the fully managed connector $connector_name ?" | ||
check_if_continue | ||
|
||
playground connector delete --connector $connector_name |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
listener 1883 | ||
persistence false | ||
log_dest stdout | ||
allow_anonymous false | ||
connection_messages true | ||
cafile /tmp/ca.crt | ||
keyfile /tmp/server.key | ||
certfile /tmp/server.crt | ||
require_certificate true | ||
tls_version tlsv1.2 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,80 @@ | ||
#!/bin/bash | ||
|
||
#set -o nounset \ | ||
# -o errexit \ | ||
# -o verbose \ | ||
# -o xtrace | ||
|
||
# Cleanup files | ||
rm -f /tmp/*.crt /tmp/*.csr /tmp/*_creds /tmp/*.jks /tmp/*.srl /tmp/*.key /tmp/*.pem /tmp/*.der /tmp/*.p12 /tmp/extfile | ||
|
||
# Generate CA key | ||
openssl req -new -x509 -keyout /tmp/snakeoil-ca-1.key -out /tmp/snakeoil-ca-1.crt -days 365 -subj '/CN=ca1.test.confluent.io/OU=TEST/O=CONFLUENT/L=PaloAlto/ST=Ca/C=US' -passin pass:confluent -passout pass:confluent | ||
|
||
for i in connect mosquitto | ||
do | ||
echo "------------------------------- $i -------------------------------" | ||
|
||
# Create host keystore | ||
keytool -genkey -noprompt \ | ||
-alias $i \ | ||
-dname "CN=$i,OU=TEST,O=CONFLUENT,L=PaloAlto,S=Ca,C=US" \ | ||
-ext "SAN=dns:$i,dns:localhost" \ | ||
-keystore /tmp/kafka.$i.keystore.jks \ | ||
-keyalg RSA \ | ||
-storepass confluent \ | ||
-keypass confluent \ | ||
-storetype pkcs12 | ||
|
||
# Create the certificate signing request (CSR) | ||
keytool -keystore /tmp/kafka.$i.keystore.jks -alias $i -certreq -file /tmp/$i.csr -storepass confluent -keypass confluent -ext "SAN=dns:$i,dns:localhost" | ||
#openssl req -in $i.csr -text -noout | ||
|
||
cat << EOF > /tmp/extfile | ||
[req] | ||
distinguished_name = req_distinguished_name | ||
x509_extensions = v3_req | ||
prompt = no | ||
[req_distinguished_name] | ||
CN = $i | ||
[v3_req] | ||
subjectAltName = @alt_names | ||
[alt_names] | ||
DNS.1 = $i | ||
DNS.2 = localhost | ||
EOF | ||
# Sign the host certificate with the certificate authority (CA) | ||
openssl x509 -req -CA /tmp/snakeoil-ca-1.crt -CAkey /tmp/snakeoil-ca-1.key -in /tmp/$i.csr -out /tmp/$i-ca1-signed.crt -days 9999 -CAcreateserial -passin pass:confluent -extensions v3_req -extfile /tmp/extfile | ||
|
||
#openssl x509 -noout -text -in $i-ca1-signed.crt | ||
|
||
# Sign and import the CA cert into the keystore | ||
keytool -noprompt -keystore /tmp/kafka.$i.keystore.jks -alias CARoot -import -file /tmp/snakeoil-ca-1.crt -storepass confluent -keypass confluent | ||
#keytool -list -v -keystore kafka.$i.keystore.jks -storepass confluent | ||
|
||
# Sign and import the host certificate into the keystore | ||
keytool -noprompt -keystore /tmp/kafka.$i.keystore.jks -alias $i -import -file /tmp/$i-ca1-signed.crt -storepass confluent -keypass confluent -ext "SAN=dns:$i,dns:localhost" | ||
#keytool -list -v -keystore kafka.$i.keystore.jks -storepass confluent | ||
|
||
# Create truststore and import the CA cert | ||
keytool -noprompt -keystore /tmp/kafka.$i.truststore.jks -alias CARoot -import -file /tmp/snakeoil-ca-1.crt -storepass confluent -keypass confluent | ||
|
||
# Save creds | ||
echo "confluent" > ${i}_sslkey_creds | ||
echo "confluent" > ${i}_keystore_creds | ||
echo "confluent" > ${i}_truststore_creds | ||
|
||
# Create pem files and keys used for Schema Registry HTTPS testing | ||
# openssl x509 -noout -modulus -in client.certificate.pem | openssl md5 | ||
# openssl rsa -noout -modulus -in client.key | openssl md5 | ||
# log "GET /" | openssl s_client -connect localhost:8081/subjects -cert client.certificate.pem -key client.key -tls1 | ||
keytool -export -alias $i -file /tmp/$i.der -keystore /tmp/kafka.$i.keystore.jks -storepass confluent | ||
openssl x509 -inform der -in /tmp/$i.der -out /tmp/$i.certificate.pem | ||
keytool -importkeystore -srckeystore /tmp/kafka.$i.keystore.jks -destkeystore /tmp/$i.keystore.p12 -deststoretype PKCS12 -deststorepass confluent -srcstorepass confluent -noprompt | ||
openssl pkcs12 -in /tmp/$i.keystore.p12 -nodes -nocerts -out /tmp/$i.key -passin pass:confluent | ||
|
||
|
||
cacerts_path="$(readlink -f /usr/bin/java | sed "s:bin/java::")lib/security/cacerts" | ||
keytool -noprompt -destkeystore /tmp/kafka.$i.truststore.jks -importkeystore -srckeystore $cacerts_path -srcstorepass changeit -deststorepass confluent | ||
|
||
done |