[HOT FIX] 01312019

helk ELK
Updated to version 6.5.4

helk-logstash
fix #156
+ Pipeline Updated
++ More security events
++ Reduced regex complexity to split process paths to process names
++ Enabled Kafka output again for Win Security and Win Sysmon logs
++ Added more win security conversion events

helk-elastalert
fix #157
fix #159

ELK:
+ Consolidated ELK scripts to one per container instead of trial and basic

helk-sigma
+ Updated own fork

helk-jupyter
+ Updated Elastic ES-Hadoop to 6.5.4

helk-jupyter
+ jupyterlab-manager widgets
+ Updated pandas 0.24.0
+ Updated altair 2.3.0

Author: Cyb3rWard0g

docker/helk-elastalert/rules/cobalt_strike_msagent.yml

@@ -12,65 +12,58 @@
 if [[ -z "$ES_HOST" ]]; then
   ES_HOST=helk-elasticsearch
 fi
-echo "[HELK-DOCKER-INSTALLATION-INFO] Setting Elasticsearch server name to $ES_HOST"
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Setting Elasticsearch server name to $ES_HOST"
 
 if [[ -z "$ES_PORT" ]]; then
   ES_PORT=9200
 fi
-echo "[HELK-DOCKER-INSTALLATION-INFO] Setting Elasticsearch server port to $ES_PORT"
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Setting Elasticsearch server port to $ES_PORT"
 
-if [[ "$ELASTIC_USERNAME" ]] && [[ "$ES_PASSWORD" ]]; then
+if [[ -n "$ELASTIC_PASSWORD" ]]; then
+    if [[ -z "$ELASTIC_USERNAME" ]]; then
+        ELASTIC_USERNAME=elastic
+    fi
     echo "es_username: $ELASTIC_USERNAME" >> $ESALERT_HOME/config.yaml
     echo "es_password: $ELASTIC_PASSWORD" >> $ESALERT_HOME/config.yaml
-    echo "[HELK-DOCKER-INSTALLATION-INFO] Setting Elasticsearch username to $ELASTIC_USERNAME"
-    echo "[HELK-DOCKER-INSTALLATION-INFO] Setting Elasticsearch password to $ELASTIC_PASSWORD"
+    echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Setting Elasticsearch username to $ELASTIC_USERNAME"
+    echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Setting Elasticsearch password to $ELASTIC_PASSWORD"
     ELASTICSEARCH_ACCESS=http://$ELASTIC_USERNAME:"$ELASTIC_PASSWORD"@$ES_HOST:$ES_PORT
-    if [[ "$KIBANA_HOST" ]] && [[ "$KIBANA_PORT" ]]; then
-        KIBANA=$KIBANA_HOST:$KIBANA_PORT
-    else
-        exit 1
-    fi
 else
     ELASTICSEARCH_ACCESS=http://$ES_HOST:$ES_PORT
 fi
 
 # *********** Update Elastalert Config ******************
-echo "[HELK-DOCKER-INSTALLATION-INFO] Updating Elastalert main config.."
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Updating Elastalert main config.."
 sed -i "s/^es_host\:.*$/es_host\: ${ES_HOST}/g" $ESALERT_HOME/config.yaml
 sed -i "s/^es_port\:.*$/es_port\: ${ES_PORT}/g" $ESALERT_HOME/config.yaml
 
 # *********** Check if Elasticsearch is up ***************
-echo "[HELK-DOCKER-INSTALLATION-INFO] Waiting for elasticsearch URI to be accessible.."
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Waiting for elasticsearch URI to be accessible.."
 until curl -s $ES_HOST:$ES_PORT -o /dev/null; do
     sleep 1
 done
 
 # *********** Creating Elastalert Status Index ***************
 response_code=$(curl -s -o /dev/null -w "%{http_code}" $ELASTICSEARCH_ACCESS/elastalert_status)
 if [[ $response_code == 404 ]]; then
-    echo "[HELK-DOCKER-INSTALLATION-INFO] Creating Elastalert index.."
-    if [[ "$ELASTIC_PASSWORD" ]]; then
-        # *********** Waiting for Kibana port to be up ***************
-        echo "[++] Checking to see if kibana port is up..."
-        until curl -s $KIBANA -o /dev/null; do
-            sleep 1
-        done
-        elastalert-create-index --host $ES_HOST --port $ES_PORT --username $ELASTIC_USERNAME--password $ELASTIC_PASSWORD --no-auth --no-ssl --url-prefix '' --old-index ''
+    echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Creating Elastalert index.."
+    if [[ -n "$ELASTIC_PASSWORD" ]]; then
+        elastalert-create-index --host $ES_HOST --port $ES_PORT --username $ELASTIC_USERNAME --password $ELASTIC_PASSWORD --no-auth --no-ssl --url-prefix '' --old-index ''
     else
         elastalert-create-index --host $ES_HOST --port $ES_PORT --no-auth --no-ssl --url-prefix '' --old-index ''
     fi
 else
-    echo "[HELK-DOCKER-INSTALLATION-INFO] Elastalert index already exists"
+    echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Elastalert index already exists"
 fi
 
 # *********** Transform SIGMA Rules to Elastalert Signatures *************
-echo "[HELK-DOCKER-INSTALLATION-INFO] Executing pull-sigma.sh script.."
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Executing pull-sigma.sh script.."
 /etc/elastalert/pull-sigma.sh
 
 # *********** Setting Slack Integration *************
 rule_counter=0
 if [[ "$SLACK_WEBHOOK_URL" ]]; then
-    echo "[HELK-DOCKER-INSTALLATION-INFO] Setting Slack webhook url to $SLACK_WEBHOOK_URL.."
+    echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Setting Slack webhook url to $SLACK_WEBHOOK_URL.."
     for er in $ESALERT_HOME/rules/*; do
         priority=$(sed -n -e 's/^priority: //p' $er)
         if [[ $priority = "1" ]]; then
@@ -96,5 +89,5 @@ if [[ "$SLACK_WEBHOOK_URL" ]]; then
     echo " "
 fi
 
-echo "[HELK-DOCKER-INSTALLATION-INFO] Starting Elastalert.."
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Starting Elastalert.."
 exec "$@"

docker/helk-elastalert/rules/cobalt_strike_rundll32.yml

@@ -10,7 +10,7 @@
 cd $ESALERT_SIGMA_HOME
 
 # ******* Check if Elastalert rules folder has SIGMA rules ************
-echo "[+++] Checking if Elastalert rules folder has SIGMA rules.."
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Checking if Elastalert rules folder has SIGMA rules.."
 if ls $ESALERT_HOME/rules/ | grep -v '^helk_' >/dev/null 2>&1; then
     echo "[+++++] SIGMA rules available in rules folder.."
     SIGMA_RULES_AVAILABLE=YES
@@ -19,11 +19,11 @@ else
 fi
 
 # ******* Check if local SIGMA repo needs update *************
-echo "[+++] Fetch updates for SIGMA remote.."
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Fetch updates for SIGMA remote.."
 git remote update
 
 # Reference: https://stackoverflow.com/a/3278427
-echo "[+++] Checking to see if local SIGMA repo is up to date or not.."
+echo "[HELK-ELASTALERT-DOCKER-INSTALLATION-INFO] Checking to see if local SIGMA repo is up to date or not.."
 UPSTREAM=${1:-'@{u}'}
 LOCAL=$(git rev-parse @)
 REMOTE=$(git rev-parse "$UPSTREAM")
@@ -85,7 +85,7 @@ find $ESALERT_HOME/rules/ -type f -name 'sigma_sysmon_powershell_suspicious_para
 
 # ******** Deleting Empty Files ***********
 echo " "
-echo "\Removing empty files.."
+echo "Removing empty files.."
 echo "-------------------------"
 rule_counter=0
 for ef in $ESALERT_HOME/rules/* ; do 

docker/helk-elastalert/rules/psexec_psh.yml

@@ -6,6 +6,6 @@
 # References: 
 # https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html
 
-FROM docker.elastic.co/elasticsearch/elasticsearch:6.5.3
+FROM docker.elastic.co/elasticsearch/elasticsearch:6.5.4
 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
 LABEL description="Dockerfile base for the HELK Elasticsearch."

docker/helk-elastalert/rules/whoami.yml

@@ -6,7 +6,7 @@
 # Author: Roberto Rodriguez (@Cyb3rWard0g)
 # License: GPL-3.0
 
-# *********** Looking for ES ***************
+# *********** Setting ES_JAVA_OPTS ***************
 if [[ -z "$ES_JAVA_OPTS" ]]; then
     ES_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024/2}' /proc/meminfo)
     if [ $ES_MEMORY -gt 31 ]; then
@@ -16,11 +16,20 @@ if [[ -z "$ES_JAVA_OPTS" ]]; then
 fi
 echo "[HELK-ES-DOCKER-INSTALLATION-INFO] Setting ES_JAVA_OPTS to $ES_JAVA_OPTS"
 
-# *********** HELK ES Password ***************
-if [[ -z "$ELASTIC_PASSWORD" ]]; then
-  export ELASTIC_PASSWORD=elasticpassword
+# ******** Checking License Type ***************
+ENVIRONMENT_VARIABLES=$(env)
+XPACK_LICENSE_TYPE="$(echo $ENVIRONMENT_VARIABLES | grep -oE 'xpack.license.self_generated.type=[^ ]*' | sed s/.*=//)"
+
+# ******** Set Trial License Variables ***************
+if [[ $XPACK_LICENSE_TYPE == "trial" ]]; then
+  # *********** HELK ES Password ***************
+  if [[ -z "$ELASTIC_PASSWORD" ]]; then
+    export ELASTIC_PASSWORD=elasticpassword
+  fi
+  echo "[HELK-ES-DOCKER-INSTALLATION-INFO] Setting Elastic password to $ELASTIC_PASSWORD"
 fi
-echo "[HELK-ES-DOCKER-INSTALLATION-INFO] Setting Elastic password to $ELASTIC_PASSWORD"
+
+echo "[HELK-ES-DOCKER-INSTALLATION-INFO] Setting Elastic license to $XPACK_LICENSE_TYPE"
 
 # ********** Starting Elasticsearch *****************
 echo "[HELK-ES-DOCKER-INSTALLATION-INFO] Running docker-entrypoint script.."

docker/helk-elastalert/scripts/elastalert-entrypoint.sh

@@ -11,7 +11,7 @@ ENV DEBIAN_FRONTEND noninteractive
 
 # *********** Setting Environment Variables ***************
 ENV JUPYTER_DIR=/opt/helk/jupyter
-ENV ESHADOOP_VERSION=6.5.3
+ENV ESHADOOP_VERSION=6.5.4
 ENV POSTGRESQL_VERSION=42.2.5
 
 # *********** Installing Prerequisites ***************
@@ -23,18 +23,20 @@ RUN apt-get update -qq \
   libxml2-dev libxslt1-dev zlib1g-dev postgresql \
   # ********** Install PIP Packages ************
   && sudo pip3 install --upgrade pip \
-  && pip3 install pandas==0.23.4 \
-  altair==2.2.2 \
+  && pip3 install pandas==0.24.0 \
+  altair==2.3.0 \
   jupyter==1.0.0 \
   jupyterlab==0.35.4 \
   jupyterhub==0.9.4 \
+  ipywidgets==7.4.2 \
   # *********** Setting Jupyter Hub & Jupyter **********************
   && curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - \
   && apt-get install -y --no-install-recommends nodejs \
   && npm install --production -g configurable-http-proxy \
   && jupyter labextension install @jupyterlab/[email protected] \
   && jupyter labextension install @jupyterlab/[email protected] \
   && jupyter labextension install @mflevine/[email protected] \
+  && jupyter labextension install @jupyter-widgets/[email protected] \
   && bash -c 'mkdir -pv /opt/helk/{es-hadoop,jupyter,jupyterhub}' \
   && mkdir -v /usr/local/share/jupyter/kernels/pyspark3 \
   && mkdir -v /var/log/spark \