Skip to content

Commit 634e24e

Browse files
Cyb3rWard0gCyb3rWard0g
committed
HELK v0.1.3-alpha08032018
All + Moved all to docker folder. Getting ready to start sharing other ways to deploy helk (terraform & Packer maybe) Compose-files + Basic & Trial Elastic Subscriptions available now and can be automatically managed via the helk_install script ELK Version : 6.3.2 Elasticsearch + Set 4GB for ES_JAVA_OPTS by default allowing the modification of it via docker-compose and calculating half of the host memory if it is not set + Added Entrypoint script and using docker-entrypoint to start ES Logstash + Big Pipeline Update by Nate Guagenti (@neu5ron) ++better cli & file name searching ++”dst_ip_public:true” filter out all rfc1918/non-routable ++Geo ASName ++Identification of 16+ windows IP fields ++Arrayed IPs support ++IPv6&IPv4 differentiation ++removing “-“ values and MORE!!! ++ THANK YOU SO MUCH NATE!!! ++ PR: #93 + Added entrypoint script to push new output_templates straight to Elasticsearch per Nate's recommendation + Starting Logstash now with docker-entrypoint + "event_data" is now taken out of winlogbeat logs to allow integration with nxlog (sauce added by Nate Guagenti (@neu5ron) Kibana + Kibana yml file updated to allow a longer time for timeout Nginx: + it handles communications to Kibana and Jupyterhub via port 443 SSL + certificate and key get created at build time + Nate added several settings to improve the way how nginx operates Jupyterhub + Multiple users and mulitple notebooks open at the same time are possible now + Jupytehub now has 3 users hunter1,hunter2.hunter3 and password patterh is <user>P@ssw0rd! + Every notebook created is also JupyterLab + Updated ES-Hadoop 6.3.2 Kafka Update + 1.1.1 Update Spark Master + Brokers + reduce memory for brokers by default to 512m Resources: + Added new images for Wiki
1 parent c7af8e4 commit 634e24e

File tree

164 files changed

+5317
-1599
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

164 files changed

+5317
-1599
lines changed

‎docker-compose-elk-basic.yml‎ renamed to ‎docker/docker-compose-elk-basic.yml‎

Lines changed: 40 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,15 @@ version: '3'
22

33
services:
44
helk-elasticsearch:
5-
image: docker.elastic.co/elasticsearch/elasticsearch:6.3.1
5+
image: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
66
container_name: helk-elasticsearch
77
volumes:
88
- ./helk-elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
99
- esdata:/usr/share/elasticsearch/data
10+
- ./helk-elasticsearch/scripts:/usr/share/elasticsearch/scripts
11+
entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh
1012
environment:
11-
- "ES_JAVA_OPTS=-Xms6g -Xmx6g"
13+
- "ES_JAVA_OPTS=-Xms4g -Xmx4g"
1214
ulimits:
1315
memlock:
1416
soft: -1
@@ -19,15 +21,17 @@ services:
1921
aliases:
2022
- helk_elasticsearch.hunt.local
2123
helk-logstash:
22-
image: docker.elastic.co/logstash/logstash:6.3.1
24+
image: docker.elastic.co/logstash/logstash:6.3.2
2325
container_name: helk-logstash
2426
volumes:
2527
- ./helk-logstash/logstash.yml:/usr/share/logstash/config/logstash.yml
2628
- ./helk-logstash/pipeline:/usr/share/logstash/pipeline
2729
- ./helk-logstash/output_templates:/usr/share/logstash/output_templates
2830
- ./helk-logstash/enrichments/cti:/usr/share/logstash/cti
31+
- ./helk-logstash/scripts:/usr/share/logstash/scripts
2932
environment:
30-
- "LS_JAVA_OPTS=-Xms2g -Xmx2g"
33+
- "LS_JAVA_OPTS=-Xms1g -Xmx1g"
34+
entrypoint: /usr/share/logstash/scripts/logstash-entrypoint.sh
3135
restart: always
3236
depends_on:
3337
- helk-elasticsearch
@@ -36,7 +40,7 @@ services:
3640
aliases:
3741
- helk_logstash.hunt.local
3842
helk-kibana:
39-
image: docker.elastic.co/kibana/kibana:6.3.1
43+
image: docker.elastic.co/kibana/kibana:6.3.2
4044
container_name: helk-kibana
4145
volumes:
4246
- ./helk-kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
@@ -51,26 +55,41 @@ services:
5155
aliases:
5256
- helk_kibana.hunt.local
5357
helk-nginx:
54-
image: cyb3rward0g/helk-nginx:0.0.3
58+
image: cyb3rward0g/helk-nginx:0.0.6
5559
container_name: helk-nginx
5660
volumes:
5761
- ./helk-nginx/htpasswd.users:/etc/nginx/htpasswd.users
5862
- ./helk-nginx/default:/etc/nginx/sites-available/default
63+
- ./helk-nginx/scripts/:/opt/helk/scripts/
64+
entrypoint: /opt/helk/scripts/nginx-entrypoint.sh
5965
ports:
6066
- "80:80"
67+
- "443:443"
6168
restart: always
6269
depends_on:
6370
- helk-kibana
6471
networks:
6572
helk:
6673
aliases:
6774
- helk_nginx.hunt.local
75+
helk-jupyter:
76+
image: cyb3rward0g/helk-jupyter:0.0.4
77+
container_name: helk-jupyter
78+
restart: always
79+
depends_on:
80+
- helk-nginx
81+
networks:
82+
helk:
83+
aliases:
84+
- helk_jupyter.hunt.local
6885
helk-spark-master:
69-
image: cyb3rward0g/helk-spark-master:2.3.1
86+
image: cyb3rward0g/helk-spark-master:2.3.1-a
7087
container_name: helk-spark-master
88+
environment:
89+
- SPARK_MASTER_PORT=7077
90+
- SPARK_MASTER_WEBUI_PORT=8080
7191
ports:
7292
- "8080:8080"
73-
- "7077:7077"
7493
restart: always
7594
depends_on:
7695
- helk-elasticsearch
@@ -79,11 +98,13 @@ services:
7998
aliases:
8099
- helk_spark_master.hunt.local
81100
helk-spark-worker:
82-
image: cyb3rward0g/helk-spark-worker:2.3.1
101+
image: cyb3rward0g/helk-spark-worker:2.3.1-a
83102
container_name: helk-spark-worker
84103
environment:
85-
- SPARK_WORKER_MEMORY=1g
104+
- SPARK_MASTER=spark://helk-spark-master:7077
105+
- SPARK_WORKER_MEMORY=512m
86106
- SPARK_WORKER_WEBUI_PORT=8081
107+
- SPARK_WORKER_PORT=42950
87108
ports:
88109
- "8081:8081"
89110
restart: always
@@ -94,11 +115,13 @@ services:
94115
aliases:
95116
- helk_spark_worker.hunt.local
96117
helk-spark-worker2:
97-
image: cyb3rward0g/helk-spark-worker:2.3.1
118+
image: cyb3rward0g/helk-spark-worker:2.3.1-a
98119
container_name: helk-spark-worker2
99120
environment:
100-
- SPARK_WORKER_MEMORY=1g
121+
- SPARK_MASTER=spark://helk-spark-master:7077
122+
- SPARK_WORKER_MEMORY=512m
101123
- SPARK_WORKER_WEBUI_PORT=8082
124+
- SPARK_WORKER_PORT=42951
102125
ports:
103126
- "8082:8082"
104127
restart: always
@@ -108,33 +131,20 @@ services:
108131
helk:
109132
aliases:
110133
- helk_spark_worker2.hunt.local
111-
helk-jupyter:
112-
image: cyb3rward0g/helk-jupyter:0.0.2
113-
container_name: helk-jupyter
114-
ports:
115-
- "8880:8880"
116-
- "4040-4050:4040-4050"
117-
restart: always
118-
depends_on:
119-
- helk-kibana
120-
networks:
121-
helk:
122-
aliases:
123-
- helk_jupyter.hunt.local
124134
helk-zookeeper:
125-
image: cyb3rward0g/helk-zookeeper:3.4.10
135+
image: cyb3rward0g/helk-zookeeper:1.1.1
126136
container_name: helk-zookeeper
127137
ports:
128138
- "2181:2181"
129139
restart: always
130140
depends_on:
131-
- helk-elasticsearch
141+
- helk-kibana
132142
networks:
133143
helk:
134144
aliases:
135145
- helk_zookeeper.hunt.local
136146
helk-kafka-broker:
137-
image: cyb3rward0g/helk-kafka-broker:1.1.0
147+
image: cyb3rward0g/helk-kafka-broker:1.1.1
138148
container_name: helk-kafka-broker
139149
restart: always
140150
depends_on:
@@ -153,7 +163,7 @@ services:
153163
aliases:
154164
- helk_kafka_broker.hunt.local
155165
helk-kafka-broker2:
156-
image: cyb3rward0g/helk-kafka-broker:1.1.0
166+
image: cyb3rward0g/helk-kafka-broker:1.1.1
157167
container_name: helk-kafka-broker2
158168
restart: always
159169
depends_on:
@@ -170,7 +180,7 @@ services:
170180
networks:
171181
helk:
172182
aliases:
173-
- helk_kafka_broker.hunt.local
183+
- helk_kafka_broker2.hunt.local
174184
helk-sigma:
175185
image: thomaspatzke/helk-sigma
176186
container_name: helk-sigma

‎docker-compose-elk-trial.yml‎ renamed to ‎docker/docker-compose-elk-trial.yml‎

Lines changed: 37 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@ services:
77
volumes:
88
- ./helk-elasticsearch/trial/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
99
- esdata:/usr/share/elasticsearch/data
10+
- ./helk-elasticsearch/scripts:/usr/share/elasticsearch/scripts
11+
entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh
1012
environment:
11-
- "ES_JAVA_OPTS=-Xms6g -Xmx6g"
13+
- "ES_JAVA_OPTS=-Xms4g -Xmx4g"
1214
ulimits:
1315
memlock:
1416
soft: -1
@@ -19,15 +21,17 @@ services:
1921
aliases:
2022
- helk_elasticsearch.hunt.local
2123
helk-logstash:
22-
image: docker.elastic.co/logstash/logstash:6.3.1
24+
image: docker.elastic.co/logstash/logstash:6.3.2
2325
container_name: helk-logstash
2426
volumes:
2527
- ./helk-logstash/trial/logstash.yml:/usr/share/logstash/config/logstash.yml
2628
- ./helk-logstash/trial/pipeline:/usr/share/logstash/pipeline
2729
- ./helk-logstash/output_templates:/usr/share/logstash/output_templates
2830
- ./helk-logstash/enrichments/cti:/usr/share/logstash/cti
31+
- ./helk-logstash/trial/scripts:/usr/share/logstash/scripts
2932
environment:
30-
- "LS_JAVA_OPTS=-Xms2g -Xmx2g"
33+
- "LS_JAVA_OPTS=-Xms1g -Xmx1g"
34+
entrypoint: /usr/share/logstash/scripts/logstash-entrypoint.sh
3135
restart: always
3236
depends_on:
3337
- helk-elasticsearch
@@ -36,7 +40,7 @@ services:
3640
aliases:
3741
- helk_logstash.hunt.local
3842
helk-kibana:
39-
image: docker.elastic.co/kibana/kibana:6.3.1
43+
image: docker.elastic.co/kibana/kibana:6.3.2
4044
container_name: helk-kibana
4145
volumes:
4246
- ./helk-kibana/trial/kibana.yml:/usr/share/kibana/config/kibana.yml
@@ -51,25 +55,40 @@ services:
5155
aliases:
5256
- helk_kibana.hunt.local
5357
helk-nginx:
54-
image: cyb3rward0g/helk-nginx:0.0.3
58+
image: cyb3rward0g/helk-nginx:0.0.6
5559
container_name: helk-nginx
5660
volumes:
5761
- ./helk-nginx/trial/default:/etc/nginx/sites-available/default
62+
- ./helk-nginx/scripts/:/opt/helk/scripts/
63+
entrypoint: /opt/helk/scripts/nginx-entrypoint.sh
5864
ports:
5965
- "80:80"
66+
- "443:443"
6067
restart: always
6168
depends_on:
6269
- helk-kibana
6370
networks:
6471
helk:
6572
aliases:
6673
- helk_nginx.hunt.local
74+
helk-jupyter:
75+
image: cyb3rward0g/helk-jupyter:0.0.4
76+
container_name: helk-jupyter
77+
restart: always
78+
depends_on:
79+
- helk-nginx
80+
networks:
81+
helk:
82+
aliases:
83+
- helk_jupyter.hunt.local
6784
helk-spark-master:
68-
image: cyb3rward0g/helk-spark-master:2.3.1
85+
image: cyb3rward0g/helk-spark-master:2.3.1-a
6986
container_name: helk-spark-master
87+
environment:
88+
- SPARK_MASTER_PORT=7077
89+
- SPARK_MASTER_WEBUI_PORT=8080
7090
ports:
7191
- "8080:8080"
72-
- "7077:7077"
7392
restart: always
7493
depends_on:
7594
- helk-elasticsearch
@@ -78,11 +97,13 @@ services:
7897
aliases:
7998
- helk_spark_master.hunt.local
8099
helk-spark-worker:
81-
image: cyb3rward0g/helk-spark-worker:2.3.1
100+
image: cyb3rward0g/helk-spark-worker:2.3.1-a
82101
container_name: helk-spark-worker
83102
environment:
84-
- SPARK_WORKER_MEMORY=1g
103+
- SPARK_MASTER=spark://helk-spark-master:7077
104+
- SPARK_WORKER_MEMORY=512m
85105
- SPARK_WORKER_WEBUI_PORT=8081
106+
- SPARK_WORKER_PORT=42950
86107
ports:
87108
- "8081:8081"
88109
restart: always
@@ -93,11 +114,13 @@ services:
93114
aliases:
94115
- helk_spark_worker.hunt.local
95116
helk-spark-worker2:
96-
image: cyb3rward0g/helk-spark-worker:2.3.1
117+
image: cyb3rward0g/helk-spark-worker:2.3.1-a
97118
container_name: helk-spark-worker2
98119
environment:
99-
- SPARK_WORKER_MEMORY=1g
120+
- SPARK_MASTER=spark://helk-spark-master:7077
121+
- SPARK_WORKER_MEMORY=512m
100122
- SPARK_WORKER_WEBUI_PORT=8082
123+
- SPARK_WORKER_PORT=42951
101124
ports:
102125
- "8082:8082"
103126
restart: always
@@ -107,21 +130,8 @@ services:
107130
helk:
108131
aliases:
109132
- helk_spark_worker2.hunt.local
110-
helk-jupyter:
111-
image: cyb3rward0g/helk-jupyter:0.0.2
112-
container_name: helk-jupyter
113-
ports:
114-
- "8880:8880"
115-
- "4040-4050:4040-4050"
116-
restart: always
117-
depends_on:
118-
- helk-kibana
119-
networks:
120-
helk:
121-
aliases:
122-
- helk_jupyter.hunt.local
123133
helk-zookeeper:
124-
image: cyb3rward0g/helk-zookeeper:3.4.10
134+
image: cyb3rward0g/helk-zookeeper:1.1.1
125135
container_name: helk-zookeeper
126136
ports:
127137
- "2181:2181"
@@ -133,7 +143,7 @@ services:
133143
aliases:
134144
- helk_zookeeper.hunt.local
135145
helk-kafka-broker:
136-
image: cyb3rward0g/helk-kafka-broker:1.1.0
146+
image: cyb3rward0g/helk-kafka-broker:1.1.1
137147
container_name: helk-kafka-broker
138148
restart: always
139149
depends_on:
@@ -152,7 +162,7 @@ services:
152162
aliases:
153163
- helk_kafka_broker.hunt.local
154164
helk-kafka-broker2:
155-
image: cyb3rward0g/helk-kafka-broker:1.1.0
165+
image: cyb3rward0g/helk-kafka-broker:1.1.1
156166
container_name: helk-kafka-broker2
157167
restart: always
158168
depends_on:

‎helk-base/Dockerfile‎ renamed to ‎docker/helk-base/Dockerfile‎

File renamed without changes.

‎helk-elasticsearch/Dockerfile‎ renamed to ‎docker/helk-elasticsearch/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
# HELK script: HELK Elasticsearch Dockerfile
22
# HELK build Stage: Alpha
3-
# HELK ELK version: 6.3.1
3+
# HELK ELK version: 6.3.2
44
# Author: Roberto Rodriguez (@Cyb3rWard0g)
55
# License: GPL-3.0
66

77
# References:
88
# https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html
99

10-
FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.1
10+
FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.2
1111
LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
1212
LABEL description="Dockerfile base for the HELK Elasticsearch."

‎helk-elasticsearch/elasticsearch.yml‎ renamed to ‎docker/helk-elasticsearch/elasticsearch.yml‎

File renamed without changes.

‎docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash
2+
3+
# HELK script: elasticsearch-entrypoint.sh
4+
# HELK script description: sets elasticsearch configs and starts elasticsearch
5+
# HELK build Stage: Alpha
6+
# Author: Roberto Rodriguez (@Cyb3rWard0g)
7+
# License: GPL-3.0
8+
9+
# *********** Looking for ES ***************
10+
if [[ ! -z "$ES_JAVA_OPTS" ]]; then
11+
echo "[HELK-DOCKER-INSTALLATION-INFO] Setting ES_JAVA_OPTS to $ES_JAVA_OPTS"
12+
else
13+
# ****** Setup heap size and memory locking *****
14+
ES_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024/2}' /proc/meminfo)
15+
echo "[HELK-DOCKER-INSTALLATION-INFO] Setting ES_HEAP_SIZE to ${ES_MEMORY}.."
16+
export ES_JAVA_OPTS="-Xms${ES_MEMORY}g -Xmx${ES_MEMORY}g"
17+
fi
18+
19+
# ********** Starting Elasticsearch *****************
20+
echo "[HELK-DOCKER-INSTALLATION-INFO] Running docker-entrypoint script.."
21+
/usr/local/bin/docker-entrypoint.sh

‎helk-elasticsearch/trial/Dockerfile‎ renamed to ‎docker/helk-elasticsearch/trial/Dockerfile‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,13 @@
11
# HELK script: HELK Elasticsearch Dockerfile
22
# HELK build Stage: Alpha
3-
# HELK ELK version: 6.3.1
3+
# HELK ELK version: 6.3.2
44
# Author: Roberto Rodriguez (@Cyb3rWard0g)
55
# License: GPL-3.0
66

77
# References:
88
# https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html
99

10-
# *********** ELK Version ***************
11-
12-
FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.1
10+
FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.2
1311
LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
1412
LABEL description="Dockerfile base for the HELK Elasticsearch."
1513

‎helk-elasticsearch/trial/elasticsearch.yml‎ renamed to ‎docker/helk-elasticsearch/trial/elasticsearch.yml‎

File renamed without changes.

0 commit comments

Comments
 (0)