Skip to content

Commit 476f26f

Browse files
algolovalgolov
committed
Add initial docker-compose files and utility scripts
1 parent 26efe8a commit 476f26f

24 files changed

+595
-0
lines changed

‎.gitignore

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
target/
2+
fs/
3+
4+
### STS ###
5+
.apt_generated
6+
.classpath
7+
.factorypath
8+
.project
9+
.settings
10+
.springBeans
11+
.sts4-cache
12+
13+
### IntelliJ IDEA ###
14+
.idea
15+
!.idea/runConfigurations
16+
*.iws
17+
*.iml
18+
*.ipr
19+
20+
### NetBeans ###
21+
/nbproject/private/
22+
/build/
23+
/nbbuild/
24+
/dist/
25+
/nbdist/
26+
/.nb-gradle/
27+
/docker-compose/volumes/grafana/data/
28+
/docker-compose/volumes/kafka/broker1/
29+
/docker-compose/volumes/kafka/broker2/
30+
/docker-compose/volumes/kafka/broker3/
31+
/docker-compose/volumes/zookeeper/zk-txn-logs/
32+
/docker-compose/volumes/zookeeper/zk-data/

‎consumer-delete.sh

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/usr/bin/env bash
2+
3+
if [[ $# -eq 0 ]]; then
4+
echo "Usage: consumer-delete.sh my-group my-other-group"
5+
exit 1
6+
fi
7+
8+
KCU_CONSUMER_GROUP="${@/#/--group }"
9+
10+
docker run --net=host --rm \
11+
confluentinc/cp-kafka:${KCU_CONFLUENT_KAFKA_VERSION:-latest} \
12+
kafka-consumer-groups \
13+
--bootstrap-server ${KCU_BOOTSTRAP_SERVER:-1.1.1.3:9092,1.1.1.4:9092,1.1.1.5:9092} \
14+
--delete ${KCU_CONSUMER_GROUP}

‎consumer-desc.sh

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env bash
2+
3+
if [[ $# -eq 0 ]]; then
4+
echo "Usage: consumer-desc.sh group-name [--offsets] [--members] [--verbose] [--state]"
5+
echo "group-name: The name of consumer group."
6+
echo "--offsets: This is the default describe option and can be omitted."
7+
echo "--members: This option provides the list of all active members in the consumer group."
8+
echo "--members --verbose: On top of the information reported by the '--members' options above, "
9+
echo " this option also provides the partitions assigned to each member."
10+
echo "--state: This option provides useful group-level information."
11+
exit 1
12+
fi
13+
14+
docker run --net=host \
15+
--rm confluentinc/cp-kafka:${KCU_CONFLUENT_KAFKA_VERSION:-latest} \
16+
kafka-consumer-groups \
17+
--bootstrap-server ${KCU_BOOTSTRAP_SERVER:-1.1.1.3:9092,1.1.1.4:9092,1.1.1.5:9092} \
18+
--describe --group $@

‎consumer-list.sh

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env bash
2+
3+
if [[ $1 == *"old"* ]]
4+
then
5+
docker run --net=host --rm \
6+
confluentinc/cp-kafka:${KCU_CONFLUENT_KAFKA_VERSION:-latest} \
7+
kafka-consumer-groups \
8+
--zookeeper ${KCU_ZK_SERVER:-1.1.1.2:2181} \
9+
--list
10+
else
11+
docker run --net=host --rm \
12+
confluentinc/cp-kafka:${KCU_CONFLUENT_KAFKA_VERSION:-latest} \
13+
kafka-consumer-groups \
14+
--bootstrap-server ${KCU_BOOTSTRAP_SERVER:-1.1.1.3:9092,1.1.1.4:9092,1.1.1.5:9092} \
15+
--list
16+
fi

‎consumer-read.sh

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env bash
2+
3+
if [[ $# -eq 0 ]]; then
4+
echo "Usage: consumer-read.sh topic-name [args..]"
5+
echo "topic-name: The topic id to produce messages to"
6+
exit 1
7+
fi
8+
9+
KCU_TOPIC=${1}
10+
shift
11+
12+
docker run --net=host --rm \
13+
confluentinc/cp-kafka:${KCU_CONFLUENT_KAFKA_VERSION:-latest} \
14+
kafka-console-consumer \
15+
--bootstrap-server ${KCU_BOOTSTRAP_SERVER:-1.1.1.3:9092,1.1.1.4:9092,1.1.1.5:9092} \
16+
--topic ${KCU_TOPIC} ${@}

‎consumer-reset.sh

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/usr/bin/env bash
2+
3+
if [[ $# -lt 2 ]]
4+
then
5+
echo "Usage: consumer-reset.sh group-name topic [position]"
6+
echo "group-name: The name of consumer group."
7+
echo "topic: The name of topic."
8+
echo "position: The position to which you want to reset the offsets."
9+
echo " --to-earliest : Reset offsets to earliest offset (this is the default)."
10+
echo " --to-latest : Reset offsets to latest offset."
11+
echo " --to-datetime <String: datetime> : Reset offsets to offsets from datetime. Format: 'YYYY-MM-DDTHH:mm:SS.sss'"
12+
echo " --shift-by <Long: number-of-offsets> : Reset offsets shifting current offset by 'n', where 'n' can be positive or negative."
13+
echo " --to-current : Resets offsets to current offset."
14+
echo " --by-duration <String: duration> : Reset offsets to offset by duration from current timestamp. Format: 'PnDTnHnMnS'"
15+
echo " --to-offset : Reset offsets to a specific offset."
16+
exit 1
17+
fi
18+
19+
KCU_TOPIC=${2}
20+
KCU_CONSUMER_GROUP=${1}
21+
22+
if [[ -z "$3" ]]; then
23+
24+
KCU_POSITION="--to-earliest"
25+
26+
else
27+
28+
shift
29+
shift
30+
31+
KCU_POSITION="$@"
32+
fi
33+
34+
docker run --net=host --rm \
35+
confluentinc/cp-kafka:${KCU_CONFLUENT_KAFKA_VERSION:-latest} \
36+
kafka-consumer-groups \
37+
--bootstrap-server ${KCU_BOOTSTRAP_SERVER:-1.1.1.3:9092,1.1.1.4:9092,1.1.1.5:9092} \
38+
--reset-offsets --group ${KCU_CONSUMER_GROUP} \
39+
--topic ${KCU_TOPIC} ${KCU_POSITION} --execute

‎docker-compose/main.yml

+122
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
version: "3"
2+
services:
3+
zookeeper:
4+
image: "confluentinc/cp-zookeeper:${KCU_CONFLUENT_KAFKA_VERSION:-latest}"
5+
environment:
6+
ZOOKEEPER_SERVER_ID: 1
7+
ZOOKEEPER_CLIENT_PORT: 2181
8+
ZOOKEEPER_TICK_TIME: 2000
9+
ZOOKEEPER_INIT_LIMIT: 5
10+
ZOOKEEPER_SYNC_LIMIT: 2
11+
ZOOKEEPER_SERVERS: 1.1.1.2:2888:3888
12+
KAFKA_JMX_PORT: 9999
13+
KAFKA_JMX_HOSTNAME: 1.1.1.2
14+
KAFKA_LOG4J_ROOT_LOGLEVEL: WARN
15+
KAFKA_TOOLS_LOG4J_LOGLEVEL: WARN
16+
container_name: zookeeper
17+
volumes:
18+
- ./volumes/zookeeper/zk-data:/var/lib/zookeeper/data
19+
- ./volumes/zookeeper/zk-txn-logs:/var/lib/zookeeper/log
20+
networks:
21+
kcu_net:
22+
ipv4_address: 1.1.1.2
23+
# logging:
24+
# driver: "none"
25+
26+
kafka1:
27+
image: "confluentinc/cp-kafka:${KCU_CONFLUENT_KAFKA_VERSION:-latest}"
28+
environment:
29+
KAFKA_BROKER_ID: 1
30+
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
31+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://1.1.1.3:9092
32+
KAFKA_JMX_PORT: 9999
33+
KAFKA_JMX_HOSTNAME: 1.1.1.3
34+
KAFKA_LOG4J_ROOT_LOGLEVEL: WARN
35+
KAFKA_TOOLS_LOG4J_LOGLEVEL: WARN
36+
container_name: kafka1
37+
volumes:
38+
- ./volumes/kafka/broker1:/var/lib/kafka/data
39+
networks:
40+
kcu_net:
41+
ipv4_address: 1.1.1.3
42+
depends_on:
43+
- zookeeper
44+
# logging:
45+
# driver: "none"
46+
47+
kafka2:
48+
image: "confluentinc/cp-kafka:${KCU_CONFLUENT_KAFKA_VERSION:-latest}"
49+
environment:
50+
KAFKA_BROKER_ID: 2
51+
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
52+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://1.1.1.4:9092
53+
KAFKA_JMX_PORT: 9999
54+
KAFKA_JMX_HOSTNAME: 1.1.1.4
55+
KAFKA_LOG4J_ROOT_LOGLEVEL: WARN
56+
KAFKA_TOOLS_LOG4J_LOGLEVEL: WARN
57+
container_name: kafka2
58+
volumes:
59+
- ./volumes/kafka/broker2:/var/lib/kafka/data
60+
networks:
61+
kcu_net:
62+
ipv4_address: 1.1.1.4
63+
depends_on:
64+
- zookeeper
65+
- kafka1
66+
# logging:
67+
# driver: "none"
68+
69+
kafka3:
70+
image: "confluentinc/cp-kafka:${KCU_CONFLUENT_KAFKA_VERSION:-latest}"
71+
environment:
72+
KAFKA_BROKER_ID: 3
73+
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
74+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://1.1.1.5:9092
75+
KAFKA_JMX_PORT: 9999
76+
KAFKA_JMX_HOSTNAME: 1.1.1.5
77+
KAFKA_LOG4J_ROOT_LOGLEVEL: WARN
78+
KAFKA_TOOLS_LOG4J_LOGLEVEL: WARN
79+
container_name: kafka3
80+
volumes:
81+
- ./volumes/kafka/broker3:/var/lib/kafka/data
82+
networks:
83+
kcu_net:
84+
ipv4_address: 1.1.1.5
85+
depends_on:
86+
- zookeeper
87+
- kafka1
88+
- kafka2
89+
# logging:
90+
# driver: "none"
91+
92+
kafka-manager:
93+
image: sheepkiller/kafka-manager:latest
94+
environment:
95+
ZK_HOSTS: zookeeper:2181
96+
KM_ARGS: -Djava.net.preferIPv4Stack=true
97+
container_name: kafka-manager
98+
networks:
99+
kcu_net:
100+
ipv4_address: 1.1.1.6
101+
depends_on:
102+
- zookeeper
103+
- kafka1
104+
- kafka2
105+
- kafka3
106+
107+
consul:
108+
image: consul:latest
109+
container_name: consul
110+
networks:
111+
kcu_net:
112+
ipv4_address: 1.1.1.7
113+
logging:
114+
driver: none
115+
116+
networks:
117+
kcu_net:
118+
ipam:
119+
driver: default
120+
config:
121+
- subnet: 1.1.1.0/24
122+
# gateway: 1.1.1.254

‎docker-compose/monitor.yml

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
version: "3"
2+
services:
3+
prom:
4+
image: quay.io/prometheus/prometheus:latest
5+
volumes:
6+
- ./volumes/prometheus:/etc/prometheus
7+
command: "--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --web.enable-admin-api"
8+
depends_on:
9+
- consul
10+
container_name: prom
11+
networks:
12+
kcu_net:
13+
ipv4_address: 1.1.1.8
14+
# logging:
15+
# driver: none
16+
17+
grafana:
18+
image: grafana/grafana:latest
19+
user: ${KCU_CUR_USER}
20+
volumes:
21+
- ./volumes/grafana/data:/var/lib/grafana
22+
- ./volumes/grafana/dashboards:/var/lib/grafana/dashboards
23+
- ./volumes/grafana/provisioning/:/etc/grafana/provisioning
24+
depends_on:
25+
- prom
26+
container_name: grafana
27+
networks:
28+
kcu_net:
29+
ipv4_address: 1.1.1.9
30+
# logging:
31+
# driver: none

‎docker-compose/volumes/grafana/provisioning/dashboards/default_provider.yml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apiVersion: 1
2+
3+
providers:
4+
- name: 'default'
5+
orgId: 1
6+
folder: ''
7+
type: file
8+
disableDeletion: false
9+
updateIntervalSeconds: 10 #how often Grafana will scan for changed dashboards
10+
options:
11+
path: /var/lib/grafana/dashboards

‎docker-compose/volumes/grafana/provisioning/datasources/prometheus.yml

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# config file version
2+
apiVersion: 1
3+
4+
# list of datasources that should be deleted from the database
5+
deleteDatasources:
6+
- name: Prometheus
7+
orgId: 1
8+
9+
# list of datasources to insert/update depending
10+
# whats available in the database
11+
datasources:
12+
# <string, required> name of the datasource. Required
13+
- name: Prometheus
14+
# <string, required> datasource type. Required
15+
type: prometheus
16+
# <string, required> access mode. proxy or direct (Server or Browser in the UI). Required
17+
access: proxy
18+
# <int> org id. will default to orgId 1 if not specified
19+
orgId: 1
20+
# <string> url
21+
url: http://1.1.1.8:9090
22+
# <string> database password, if used
23+
password:
24+
# <string> database user, if used
25+
user:
26+
# <string> database name, if used
27+
database:
28+
# <bool> enable/disable basic auth
29+
basicAuth: false
30+
# <string> basic auth username
31+
basicAuthUser:
32+
# <string> basic auth password
33+
basicAuthPassword:
34+
# <bool> enable/disable with credentials headers
35+
withCredentials:
36+
# <bool> mark as default datasource. Max one per org
37+
isDefault: true
38+
# <map> fields that will be converted to json and stored in json_data
39+
jsonData:
40+
graphiteVersion: "1.1"
41+
tlsAuth: false
42+
tlsAuthWithCACert: false
43+
# <string> json object of data that will be encrypted.
44+
secureJsonData:
45+
tlsCACert: "..."
46+
tlsClientCert: "..."
47+
tlsClientKey: "..."
48+
version: 1
49+
# <bool> allow users to edit datasources from the UI.
50+
editable: true

‎docker-compose/volumes/prometheus/prometheus.yml

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
global:
2+
scrape_interval: 15s
3+
4+
scrape_configs:
5+
- job_name: 'prometheus'
6+
scrape_interval: 5s
7+
static_configs:
8+
- targets: ['localhost:9090']
9+
10+
- job_name: 'consul-spring-boot'
11+
metrics_path: '/actuator/prometheus'
12+
consul_sd_configs:
13+
- server: 'consul:8500'
14+
services: []
15+
relabel_configs:
16+
- source_labels: [__meta_consul_tags]
17+
regex: .*,springboot,.*
18+
action: keep
19+
20+
- source_labels: [__meta_consul_service]
21+
target_label: job
22+
23+
- source_labels: [__meta_consul_dc]
24+
regex: '(.*)'
25+
target_label: dc
26+
replacement: ${1}

0 commit comments

Comments
 (0)