add hive metastore change monitor script.

hivemetastore-change-monitor/bin/alertUtils.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env python
+
+import httplib, urllib, urllib2, logging, os, time, sys
+
+baseHome = os.path.abspath(sys.path[0] + '/..')
+
+alertModes1 = {"mail": "https://master.mail.netease.com/omnew/alert/sendMultiAlert", "sms": "http://bigdata0.photo.163.org:8080/sentry/sendsms.jsp"}
+alertModes = {"mail": "http://172.16.87.242:8080/omnew/alert/sendMultiAlert", "sms": "http://bigdata0.photo.163.org:8080/sentry/sendsms.jsp"}
+receivers = {"mail": "[email protected]", "sms": "13777495757"}
+
+def sendAlert(alertType, subject, body):
+        url = alertModes[alertType]
+        body_value = {"account": receivers[alertType] ,"subject": subject, "mobile": receivers["sms"], "emailMsg": body, "popoMsg": body, "mobileMsg": body, "product":"holmes_alarm", "type":7}
+
+#        print(body_value)
+
+        request = urllib2.Request(url, str(body_value))
+        request.add_header("Content-type", "application/json")
+        request.add_header("Accept", "text/plain")
+        resp = urllib2.urlopen(request )
+        logging.info( "send alters to " + receivers[alertType] + " by "+ alertType + ". Got response code " + bytes(resp.getcode()))

hivemetastore-change-monitor/bin/metastoreChange-check.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# $1:zookeeperQueum
+# $2:command
+# $3:znode path
+# $4:params
+
+metastore_change_path=$1
+
+BASE_PATH=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
+
+# get config parameters
+. ${BASE_PATH}/monitor-env.sh
+
+if [[ $# -eq 0 ]]; then
+  echo "$BASH_SOURCE:$LINENO 请输入执行参数!"
+  exit;
+fi
+# echo $1 $2 $3 $4
+
+> "$BASE_PATH"/../metastore_result.txt
+${ZOOKEEPER_CLI_PATH} -server ${ZOOKEEPER_SERVER} <<EOF
+get ${metastore_change_path}
+EOF
+>> "$BASE_PATH"/../metastore_result.txt

hivemetastore-change-monitor/bin/monitor-env.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+BASE_PATH=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
+echo "parameter: $1"
+
+export metastore_namespaces=hive-cluster3,hive-cluster4
+export keytab="$BASE_PATH"/../conf/hive.keytab
+export principal=hive/[email protected]
+export hiveserver2="hzabj-backend-hiveserver0.server.163.org:10000/default"
+export hiveserver2_principal=hive/[email protected]
+export HIVE_SERVER2_THRIFT_BIND_HOST=hadoop707.lt.163.org
+
+#for python script
+export hiveBin="/home/hadoop/hive-current/bin"
+
+
+export ZOOKEEPER_CLI_PATH="/home/hadoop/zookeeper-current/bin/zkCli.sh"
+export ZOOKEEPER_SERVER="holmes-mm-5.hz.infra.mail"
+export METASTORE_CHANGE_PATH=/hive-metastore-changelog/metastore1/maxid

hivemetastore-change-monitor/bin/monitorHiveMetastoreChange.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env python
+import os
+import time
+import logging
+import subprocess as sp
+import sys
+import alertUtils
+import logging.handlers
+
+baseHome = os.path.abspath(sys.path[0] + '/..')
+int_preMaxId=999999999
+int_currentMaxId=0
+
+
+#logging.basicConfig(level=logging.DEBUG,format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',datefmt='%a, %d %b %Y %H:%M:%S',filename=baseHome+'/logs/hiveMetastoreMonitor.log',filemode='a')
+
+def log_setup():
+    log_handler = logging.handlers.TimedRotatingFileHandler(baseHome+'/logs/hiveMetastoreMonitorChange.log',when="midnight")
+    formatter = logging.Formatter(
+        '%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
+        '%a, %d %b %Y %H:%M:%S')
+    log_handler.setFormatter(formatter)
+    logger = logging.getLogger()
+    logger.addHandler(log_handler)
+    logger.setLevel(logging.DEBUG)
+
+
+def check(metastore_change_path):
+    global int_preMaxId
+    global int_currentMaxId
+
+    logging.info('now checking hive metastore change maxid' + metastore_change_path)
+    filePreMaxid = open(baseHome + '/maxid.txt','r+')
+    preMaxId = filePreMaxid.read()
+    if preMaxId.strip() == '':
+        preMaxId = '0'
+
+    int_preMaxId = int(preMaxId)
+
+    os.system(baseHome + "/bin/metastoreChange-check.sh " + metastore_change_path)
+    f = open(baseHome + '/metastore_result.txt','r')
+    lines = f.readlines()
+    f.close()
+    lines_length = len(lines)
+    if lines_length >= 2:
+        currentMaxId = lines[lines_length-2];
+        int_currentMaxId = int(currentMaxId)
+
+        print 'int_preMaxId = %d' % int_preMaxId
+        print 'int_currentMaxId = %d' % int_currentMaxId
+        logging.info('int_preMaxId = ' + str(int_preMaxId))        
+        logging.info('int_currentMaxId = ' + str(int_currentMaxId))
+
+        if int_currentMaxId > int_preMaxId:
+            filePreMaxid.seek(0,0)
+            filePreMaxid.write(currentMaxId)
+            filePreMaxid.close()
+            return True
+        else:
+            return False
+    else:
+        return False
+
+
+def getEnvValFromScript(script):
+    proc = sp.Popen(['bash', '-c', 'source {} && env'.format(script)], stdout=sp.PIPE)
+    source_env = {tup[0].strip(): tup[1].strip() for tup in map(lambda s: s.strip().split('=', 1), proc.stdout)}
+    return source_env
+
+log_setup()
+source_env = getEnvValFromScript(baseHome + "/bin/monitor-env.sh")
+metastore_change_paths = source_env['METASTORE_CHANGE_PATH'].split(",")
+
+for change_path in metastore_change_paths:
+    if check(change_path):
+        logging.info('The hive metastore change ' + change_path + ', is so good ~ ')
+    else:
+        logging.error('The hive metastore change ' + change_path + ', is dead! Send alert')
+        alertUtils.sendAlert("mail","MetaStoreChange error", change_path + "int_preMaxId=" + str(int_preMaxId) + ", int_currentMaxId=" + str(int_currentMaxId))
+
+logging.info(' -----------------------------\n')

hivemetastore-change-monitor/maxid.txt

@@ -0,0 +1 @@
+0

hivemetastore-change-monitor/metastore_result.txt

@@ -0,0 +1,2 @@
+database_name	tab_name
+default	cn_comment_test3

hivemetastore-change-monitor/result.txt

@@ -0,0 +1,5 @@
++----------------+-------------------+--+
+| database_name  |     tab_name      |
++----------------+-------------------+--+
+| default        | cn_comment_test3  |
++----------------+-------------------+--+