From bc7c9cd80dcbb11b1127704b85ea78b7e5ef8da4 Mon Sep 17 00:00:00 2001 From: hzliuxun Date: Fri, 10 Nov 2017 19:15:00 +0800 Subject: [PATCH] add hive metastore change monitor script. --- .../bin/alertUtils.py | 22 +++++ .../bin/metastoreChange-check.sh | 25 ++++++ .../bin/monitor-env.sh | 19 +++++ .../bin/monitorHiveMetastoreChange.py | 82 +++++++++++++++++++ hivemetastore-change-monitor/maxid.txt | 1 + .../metastore_result.txt | 2 + hivemetastore-change-monitor/result.txt | 5 ++ 7 files changed, 156 insertions(+) create mode 100755 hivemetastore-change-monitor/bin/alertUtils.py create mode 100755 hivemetastore-change-monitor/bin/metastoreChange-check.sh create mode 100755 hivemetastore-change-monitor/bin/monitor-env.sh create mode 100755 hivemetastore-change-monitor/bin/monitorHiveMetastoreChange.py create mode 100644 hivemetastore-change-monitor/maxid.txt create mode 100644 hivemetastore-change-monitor/metastore_result.txt create mode 100644 hivemetastore-change-monitor/result.txt diff --git a/hivemetastore-change-monitor/bin/alertUtils.py b/hivemetastore-change-monitor/bin/alertUtils.py new file mode 100755 index 00000000..6de6898f --- /dev/null +++ b/hivemetastore-change-monitor/bin/alertUtils.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + +import httplib, urllib, urllib2, logging, os, time, sys + +baseHome = os.path.abspath(sys.path[0] + '/..') + +alertModes1 = {"mail": "https://master.mail.netease.com/omnew/alert/sendMultiAlert", "sms": "http://bigdata0.photo.163.org:8080/sentry/sendsms.jsp"} +alertModes = {"mail": "http://172.16.87.242:8080/omnew/alert/sendMultiAlert", "sms": "http://bigdata0.photo.163.org:8080/sentry/sendsms.jsp"} +receivers = {"mail": "hzliuxun@corp.netease.com", "sms": "13777495757"} + +def sendAlert(alertType, subject, body): + url = alertModes[alertType] + body_value = {"account": receivers[alertType] ,"subject": subject, "mobile": receivers["sms"], "emailMsg": body, "popoMsg": body, "mobileMsg": body, "product":"holmes_alarm", "type":7} + +# print(body_value) + + request = urllib2.Request(url, str(body_value)) + request.add_header("Content-type", "application/json") + request.add_header("Accept", "text/plain") + resp = urllib2.urlopen(request ) + logging.info( "send alters to " + receivers[alertType] + " by "+ alertType + ". Got response code " + bytes(resp.getcode())) \ No newline at end of file diff --git a/hivemetastore-change-monitor/bin/metastoreChange-check.sh b/hivemetastore-change-monitor/bin/metastoreChange-check.sh new file mode 100755 index 00000000..30d39fbe --- /dev/null +++ b/hivemetastore-change-monitor/bin/metastoreChange-check.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# $1:zookeeperQueum +# $2:command +# $3:znode path +# $4:params + +metastore_change_path=$1 + +BASE_PATH=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd) + +# get config parameters +. ${BASE_PATH}/monitor-env.sh + +if [[ $# -eq 0 ]]; then + echo "$BASH_SOURCE:$LINENO 请输入执行参数!" + exit; +fi +# echo $1 $2 $3 $4 + +> "$BASE_PATH"/../metastore_result.txt +${ZOOKEEPER_CLI_PATH} -server ${ZOOKEEPER_SERVER} <> "$BASE_PATH"/../metastore_result.txt diff --git a/hivemetastore-change-monitor/bin/monitor-env.sh b/hivemetastore-change-monitor/bin/monitor-env.sh new file mode 100755 index 00000000..d1a4ef2e --- /dev/null +++ b/hivemetastore-change-monitor/bin/monitor-env.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +BASE_PATH=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd) +echo "parameter: $1" + +export metastore_namespaces=hive-cluster3,hive-cluster4 +export keytab="$BASE_PATH"/../conf/hive.keytab +export principal=hive/app-20.photo.163.org@HADOOP.HZ.NETEASE.COM +export hiveserver2="hzabj-backend-hiveserver0.server.163.org:10000/default" +export hiveserver2_principal=hive/app-20.photo.163.org@HADOOP.HZ.NETEASE.COM +export HIVE_SERVER2_THRIFT_BIND_HOST=hadoop707.lt.163.org + +#for python script +export hiveBin="/home/hadoop/hive-current/bin" + + +export ZOOKEEPER_CLI_PATH="/home/hadoop/zookeeper-current/bin/zkCli.sh" +export ZOOKEEPER_SERVER="holmes-mm-5.hz.infra.mail" +export METASTORE_CHANGE_PATH=/hive-metastore-changelog/metastore1/maxid \ No newline at end of file diff --git a/hivemetastore-change-monitor/bin/monitorHiveMetastoreChange.py b/hivemetastore-change-monitor/bin/monitorHiveMetastoreChange.py new file mode 100755 index 00000000..f44d2015 --- /dev/null +++ b/hivemetastore-change-monitor/bin/monitorHiveMetastoreChange.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python +import os +import time +import logging +import subprocess as sp +import sys +import alertUtils +import logging.handlers + +baseHome = os.path.abspath(sys.path[0] + '/..') +int_preMaxId=999999999 +int_currentMaxId=0 + + +#logging.basicConfig(level=logging.DEBUG,format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',datefmt='%a, %d %b %Y %H:%M:%S',filename=baseHome+'/logs/hiveMetastoreMonitor.log',filemode='a') + +def log_setup(): + log_handler = logging.handlers.TimedRotatingFileHandler(baseHome+'/logs/hiveMetastoreMonitorChange.log',when="midnight") + formatter = logging.Formatter( + '%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', + '%a, %d %b %Y %H:%M:%S') + log_handler.setFormatter(formatter) + logger = logging.getLogger() + logger.addHandler(log_handler) + logger.setLevel(logging.DEBUG) + + +def check(metastore_change_path): + global int_preMaxId + global int_currentMaxId + + logging.info('now checking hive metastore change maxid' + metastore_change_path) + filePreMaxid = open(baseHome + '/maxid.txt','r+') + preMaxId = filePreMaxid.read() + if preMaxId.strip() == '': + preMaxId = '0' + + int_preMaxId = int(preMaxId) + + os.system(baseHome + "/bin/metastoreChange-check.sh " + metastore_change_path) + f = open(baseHome + '/metastore_result.txt','r') + lines = f.readlines() + f.close() + lines_length = len(lines) + if lines_length >= 2: + currentMaxId = lines[lines_length-2]; + int_currentMaxId = int(currentMaxId) + + print 'int_preMaxId = %d' % int_preMaxId + print 'int_currentMaxId = %d' % int_currentMaxId + logging.info('int_preMaxId = ' + str(int_preMaxId)) + logging.info('int_currentMaxId = ' + str(int_currentMaxId)) + + if int_currentMaxId > int_preMaxId: + filePreMaxid.seek(0,0) + filePreMaxid.write(currentMaxId) + filePreMaxid.close() + return True + else: + return False + else: + return False + + +def getEnvValFromScript(script): + proc = sp.Popen(['bash', '-c', 'source {} && env'.format(script)], stdout=sp.PIPE) + source_env = {tup[0].strip(): tup[1].strip() for tup in map(lambda s: s.strip().split('=', 1), proc.stdout)} + return source_env + +log_setup() +source_env = getEnvValFromScript(baseHome + "/bin/monitor-env.sh") +metastore_change_paths = source_env['METASTORE_CHANGE_PATH'].split(",") + +for change_path in metastore_change_paths: + if check(change_path): + logging.info('The hive metastore change ' + change_path + ', is so good ~ ') + else: + logging.error('The hive metastore change ' + change_path + ', is dead! Send alert') + alertUtils.sendAlert("mail","MetaStoreChange error", change_path + "int_preMaxId=" + str(int_preMaxId) + ", int_currentMaxId=" + str(int_currentMaxId)) + +logging.info(' -----------------------------\n') diff --git a/hivemetastore-change-monitor/maxid.txt b/hivemetastore-change-monitor/maxid.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/hivemetastore-change-monitor/maxid.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/hivemetastore-change-monitor/metastore_result.txt b/hivemetastore-change-monitor/metastore_result.txt new file mode 100644 index 00000000..693a8fd3 --- /dev/null +++ b/hivemetastore-change-monitor/metastore_result.txt @@ -0,0 +1,2 @@ +database_name tab_name +default cn_comment_test3 diff --git a/hivemetastore-change-monitor/result.txt b/hivemetastore-change-monitor/result.txt new file mode 100644 index 00000000..38e28d8a --- /dev/null +++ b/hivemetastore-change-monitor/result.txt @@ -0,0 +1,5 @@ ++----------------+-------------------+--+ +| database_name | tab_name | ++----------------+-------------------+--+ +| default | cn_comment_test3 | ++----------------+-------------------+--+