Skip to content

Commit

Permalink
Merge pull request #5918 from confluentinc/16092024-2
Browse files Browse the repository at this point in the history
16092024 2 (#9)
  • Loading branch information
vdesabou authored Sep 17, 2024
2 parents dc471b8 + 10e2ca2 commit 171d76b
Show file tree
Hide file tree
Showing 9 changed files with 117 additions and 30 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ jobs:
"🚀 environment/plaintext environment/2way-ssl environment/kerberos environment/ldap-authorizer-sasl-plain environment/ldap-sasl-plain environment/mdc-kerberos environment/mdc-plaintext environment/mdc-sasl-plain environment/rbac-sasl-plain environment/sasl-plain environment/sasl-scram environment/sasl-ssl environment/ssl_kerberos environment/kraft-plaintext connect/connect-snowflake-sink",
"🚀 connect/connect-mapr-sink",
"🚀 other/syslog-logstash-ksqldb other/mqtt-proxy connect/connect-jdbc-snowflake-source connect/connect-jdbc-snowflake-sink connect/connect-filestream-source connect/connect-filestream-sink connect/connect-filepulse-source connect/connect-jdbc-mariadb-source connect/connect-jdbc-mariadb-sink",
"🚀 connect/connect-jdbc-sap-hana-sink connect/connect-jdbc-sap-hana-source connect/connect-sap-hana-sink",
"🚀 connect/connect-jdbc-sap-hana-sink connect/connect-jdbc-sap-hana-source connect/connect-sap-hana-sink other/kafka-connect-jsonata",

# requiring ngrok
"🚀2️⃣ ccloud/fm-influxdb2-sink ccloud/fm-influxdb2-source ccloud/fm-jdbc-oracle19-source ccloud/fm-jdbc-oracle19-sink ccloud/fm-cdc-oracle19-source ccloud/fm-rabbitmq-source ccloud/fm-zendesk-source ccloud/fm-splunk-sink ccloud/fm-rabbitmq-sink ccloud/fm-jdbc-mysql-sink ccloud/fm-sftp-source ccloud/fm-http-sink ccloud/fm-http-v2-sink ccloud/fm-http-source ccloud/fm-http-v2-source",
Expand Down
22 changes: 13 additions & 9 deletions ccloud/fm-gcp-bigtable-sink/fully-managed-gcp-bigtable-sink.sh
Original file line number Diff line number Diff line change
Expand Up @@ -134,16 +134,20 @@ wait_for_ccloud_connector_up $connector_name 180

playground connector show-lag --connector $connector_name --max-wait 360

log "Doing gsutil authentication"
set +e
docker rm -f gcloud-config
set -e
docker run -i -v ${GCP_KEYFILE}:/tmp/keyfile.json --name gcloud-config google/cloud-sdk:latest gcloud auth activate-service-account --project ${GCP_PROJECT} --key-file /tmp/keyfile.json
if [ -z "$GITHUB_RUN_NUMBER" ]
then
# not running with github actions
log "Doing gsutil authentication"
set +e
docker rm -f gcloud-config
set -e
docker run -i -v ${GCP_KEYFILE}:/tmp/keyfile.json --name gcloud-config google/cloud-sdk:latest gcloud auth activate-service-account --project ${GCP_PROJECT} --key-file /tmp/keyfile.json

log "Verify data is in GCP BigTable"
docker run -i --volumes-from gcloud-config google/cloud-sdk:latest cbt -project $GCP_PROJECT -instance $GCP_BIGTABLE_INSTANCE read kafka_big_query_stats > /tmp/result.log 2>&1
cat /tmp/result.log
grep "Bob" /tmp/result.log
log "Verify data is in GCP BigTable"
docker run -i --volumes-from gcloud-config google/cloud-sdk:latest cbt -project $GCP_PROJECT -instance $GCP_BIGTABLE_INSTANCE read kafka_big_query_stats > /tmp/result.log 2>&1
cat /tmp/result.log
grep "Bob" /tmp/result.log
fi

log "Do you want to delete the fully managed connector $connector_name ?"
check_if_continue
Expand Down
24 changes: 14 additions & 10 deletions connect/connect-gcp-bigtable-sink/gcp-bigtable-sink-proxy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -123,13 +123,17 @@ EOF

playground connector show-lag --connector gcp-bigtable-sink --max-wait 360

log "Doing gsutil authentication"
set +e
docker rm -f gcloud-config
set -e
docker run -i -v ${GCP_KEYFILE}:/tmp/keyfile.json --name gcloud-config google/cloud-sdk:latest gcloud auth activate-service-account --project ${GCP_PROJECT} --key-file /tmp/keyfile.json

log "Verify data is in GCP BigTable"
docker run -i --volumes-from gcloud-config google/cloud-sdk:latest cbt -project $GCP_PROJECT -instance $GCP_BIGTABLE_INSTANCE read kafka_big_query_stats > /tmp/result.log 2>&1
cat /tmp/result.log
grep "Bob" /tmp/result.log
if [ -z "$GITHUB_RUN_NUMBER" ]
then
# not running with github actions
log "Doing gsutil authentication"
set +e
docker rm -f gcloud-config
set -e
docker run -i -v ${GCP_KEYFILE}:/tmp/keyfile.json --name gcloud-config google/cloud-sdk:latest gcloud auth activate-service-account --project ${GCP_PROJECT} --key-file /tmp/keyfile.json

log "Verify data is in GCP BigTable"
docker run -i --volumes-from gcloud-config google/cloud-sdk:latest cbt -project $GCP_PROJECT -instance $GCP_BIGTABLE_INSTANCE read kafka_big_query_stats > /tmp/result.log 2>&1
cat /tmp/result.log
grep "Bob" /tmp/result.log
fi
22 changes: 13 additions & 9 deletions connect/connect-gcp-bigtable-sink/gcp-bigtable-sink.sh
Original file line number Diff line number Diff line change
Expand Up @@ -117,13 +117,17 @@ EOF

playground connector show-lag --connector gcp-bigtable-sink --max-wait 360

log "Doing gsutil authentication"
set +e
docker rm -f gcloud-config
set -e
docker run -i -v ${GCP_KEYFILE}:/tmp/keyfile.json --name gcloud-config google/cloud-sdk:latest gcloud auth activate-service-account --project ${GCP_PROJECT} --key-file /tmp/keyfile.json
if [ -z "$GITHUB_RUN_NUMBER" ]
then
# not running with github actions
log "Doing gsutil authentication"
set +e
docker rm -f gcloud-config
set -e
docker run -i -v ${GCP_KEYFILE}:/tmp/keyfile.json --name gcloud-config google/cloud-sdk:latest gcloud auth activate-service-account --project ${GCP_PROJECT} --key-file /tmp/keyfile.json

log "Verify data is in GCP BigTable"
docker run -i --volumes-from gcloud-config google/cloud-sdk:latest cbt -project $GCP_PROJECT -instance $GCP_BIGTABLE_INSTANCE read kafka_big_query_stats > /tmp/result.log 2>&1
cat /tmp/result.log
grep "Bob" /tmp/result.log
log "Verify data is in GCP BigTable"
docker run -i --volumes-from gcloud-config google/cloud-sdk:latest cbt -project $GCP_PROJECT -instance $GCP_BIGTABLE_INSTANCE read kafka_big_query_stats > /tmp/result.log 2>&1
cat /tmp/result.log
grep "Bob" /tmp/result.log
fi
1 change: 1 addition & 0 deletions docs/content-template.md
Original file line number Diff line number Diff line change
Expand Up @@ -416,3 +416,4 @@ Using Multi-Data-Center setup with 🇺🇸 and 🇪🇺 clusters
- 🪦 [Recovery from schema hard deletion](https://github.com/vdesabou/kafka-docker-playground/tree/master/other/recover-from-schema-hard-deletion) ![not tested](https://img.shields.io/badge/CI-not%20tested!-violet)
- 🔍 [ksqlDB Schema Inference with ID](https://github.com/vdesabou/kafka-docker-playground/tree/master/ksqldb/schema-inference-with-id) ![not tested](https://img.shields.io/badge/CI-not%20tested!-violet)
- 💬 [MQTT Proxy](https://github.com/vdesabou/kafka-docker-playground/tree/master/other/mqtt-proxy) :other/mqtt-proxy:
- 💱 [Kafka Connect JSONata Transform](https://github.com/vdesabou/kafka-docker-playground/tree/master/other/kafka-connect-jsonata) :other/kafka-connect-jsonata:
7 changes: 7 additions & 0 deletions other/kafka-connect-jsonata/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# kafka-connect-jsonata SMT



## Objective

Quickly test [kafka-connect-jsonata](https://github.com/rayokota/kafka-connect-jsonata) SMT.
5 changes: 5 additions & 0 deletions other/kafka-connect-jsonata/docker-compose.plaintext.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
services:
connect:
environment:
CONNECT_PLUGIN_PATH: /usr/share/filestream-connectors,/usr/share/confluent-hub-components/rayokota-kafka-connect-jsonata
62 changes: 62 additions & 0 deletions other/kafka-connect-jsonata/jsonata-drop-tombstone.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#!/bin/bash
set -e

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
source ${DIR}/../../scripts/utils.sh

PLAYGROUND_ENVIRONMENT=${PLAYGROUND_ENVIRONMENT:-"plaintext"}
playground start-environment --environment "${PLAYGROUND_ENVIRONMENT}" --docker-compose-override-file "${PWD}/docker-compose.plaintext.yml"

log "Sending messages to topic filestream"
playground topic produce -t filestream --nb-messages 5 --key 1 << 'EOF'
{
"fields": [
{
"doc": "count",
"name": "count",
"type": "long"
},
{
"doc": "First Name of Customer",
"name": "first_name",
"type": "string"
},
{
"doc": "Last Name of Customer",
"name": "last_name",
"type": "string"
},
{
"doc": "Address of Customer",
"name": "address",
"type": "string"
}
],
"name": "Customer",
"namespace": "com.github.vdesabou",
"type": "record"
}
EOF

playground topic produce -t filestream --key 1 --tombstone


log "Creating FileStream Sink connector"
playground connector create-or-update --connector filestream-sink << EOF
{
"tasks.max": "1",
"connector.class": "org.apache.kafka.connect.file.FileStreamSinkConnector",
"topics": "filestream",
"file": "/tmp/output.json",
"transforms": "dropTombstone",
"transforms.dropTombstone.type": "io.yokota.kafka.connect.transform.jsonata.JsonataTransformation",
"transforms.dropTombstone.expr": "value = null ? null : \$"
}
EOF


sleep 5

log "Verify we have received the data in file"
docker exec connect cat /tmp/output.json
2 changes: 1 addition & 1 deletion scripts/cli/src/lib/utils_function.sh
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ function maybe_create_image()
export CONNECT_USER="appuser"
if [ `uname -m` = "arm64" ]
then
CONNECT_3RDPARTY_INSTALL="if [ ! -f /tmp/done ]; then yum -y install --disablerepo='Confluent*' bind-utils openssl unzip findutils net-tools nc jq which iptables libmnl krb5-workstation krb5-libs vim && yum clean all && rm -rf /var/cache/yum && rpm -i --nosignature https://koji.mbox.centos.org/kojifiles/packages/tcpdump/4.9.3/3.el8/aarch64/tcpdump-4.9.3-3.el8.aarch64.rpm && touch /tmp/done; fi"
CONNECT_3RDPARTY_INSTALL="if [ ! -f /tmp/done ]; then yum -y install --disablerepo='Confluent*' bind-utils openssl unzip findutils net-tools nc jq which iptables libmnl krb5-workstation krb5-libs vim && yum clean all && rm -rf /var/cache/yum && rpm -i --nosignature https://yum.oracle.com/repo/OracleLinux/OL8/appstream/aarch64/getPackage/tcpdump-4.9.3-3.el8.aarch64.rpm && touch /tmp/done; fi"
else
CONNECT_3RDPARTY_INSTALL="if [ ! -f /tmp/done ]; then curl https://download.rockylinux.org/pub/rocky/8/AppStream/x86_64/kickstart/Packages/t/tcpdump-4.9.3-5.el8.x86_64.rpm -o tcpdump-4.9.3-1.el8.x86_64.rpm && rpm -Uvh tcpdump-4.9.3-1.el8.x86_64.rpm && yum -y install --disablerepo='Confluent*' bind-utils openssl unzip findutils net-tools nc jq which iptables libmnl krb5-workstation krb5-libs vim && yum clean all && rm -rf /var/cache/yum && touch /tmp/done; fi"
fi
Expand Down

0 comments on commit 171d76b

Please sign in to comment.