Merge pull request #1 from elPatron/master

Adding latest and greatest stu fixes to CBS
newBrandAnalytics · Jun 24, 2014 · df6801c · df6801c
2 parents ce16c57 + 850a41f
commit df6801c
Show file tree

Hide file tree

Showing 7 changed files with 97 additions and 14 deletions.
diff --git a/config/template/boot-two b/config/template/boot-two
@@ -69,7 +69,10 @@ curl -s "http://${HYDRA_BOOT}/render/hcl?cluster=${CLUSTER_ID}&hostname=${CLUSTE
 
 # launch host local processes
 for proc in ${PROCESSES}; do
-	echo -n "fetching ${proc} start... "
-	bash <(curl -s "http://${HYDRA_BOOT}/render/start-${proc}?cluster=${CLUSTER_ID}&hostname=${CLUSTER_HOST}")
+	echo "${proc}" | tr ':' ' ' | while read procname procarg; do
+		echo -n "fetching ${procname} start... "
+		export PROC_ARG=${procarg}
+		bash <(curl -s "http://${HYDRA_BOOT}/render/start-${procname}?cluster=${CLUSTER_ID}&hostname=${CLUSTER_HOST}")
+	done
 done
 
diff --git a/config/template/me/query/query.js b/config/template/me/query/query.js
@@ -443,6 +443,9 @@ function limit(txt,chars) {
 
 /* cache entry list render */
 function renderCacheList(list,div,kill) {
+    list.sort(function(a,b) {
+      return b.startTime - a.startTime;
+    });
     var html = '<table class="render"><tr><th>';
     html += ['submit','uuid','alias','job','path','ops','run','lines','kill'].join('</th><th>')+'</th></tr>';
     for (var i=0; i<list.length; i++) {

diff --git a/config/template/me/spawn/spawn.js b/config/template/me/spawn/spawn.js
@@ -433,8 +433,10 @@ function showEdit(e,b) {
 	$(e).style.display = b ? 'block' : 'none';
 	if (b) {
 		editing = e;
+        setPollerQueue(null, true);
 	} else {
 		editing = null;
+        setPollerLive();
 	}
 }
 
@@ -650,26 +652,29 @@ function fdate(v) {
 }
 
 /* set spawn poller to queueing mode */
-function setPollerQueue(evt) {
+function setPollerQueue(evt,disable) {
 	if (queuePoller != null) {
 		clearTimeout(queuePoller);
 		queuePoller = null;
 	}
+    if (disable === true) {
+        enablePolling = false;
+        $('event_count').innerHTML = 'paused';
+    }
 	if (enablePolling) {
 		try {
-		$('event_count').innerHTML = 'queueing';
-		queuePoller = setTimeout(setPollerLive, 500);
+    		queuePoller = setTimeout(setPollerLive, 500);
+            $('event_count').innerHTML = 'queueing';
 		} catch (e) { }
-		//console.log(['set poller queueing', queuedEvents]);
 	}
 }
 
 /* set spawn poller to live mode */
 function setPollerLive() {
+    enablePolling = true;
 	if (queuePoller != null) {
 		clearTimeout(queuePoller);
 		queuePoller = null;
-		//console.log(['set poller live', queuedEvents]);
 		$('event_count').innerHTML = 'live';
 		if (queuedEvents.length > 0) {
 			var qev = queuedEvents;

diff --git a/config/template/proc b/config/template/proc
@@ -4,15 +4,20 @@
 
 function startProcessList() {
 	for proc in $*; do
-		echo -n "fetching ${proc} start... "
-		bash <(curl -s "http://${HYDRA_BOOT}/render/start-${proc}?cluster=${CLUSTER_ID}&hostname=${CLUSTER_HOST}")
+		echo "${proc}" | tr ':' ' ' | while read procname procarg; do
+			echo -n "fetching ${procname} start... "
+			export PROC_ARG=${procarg}
+			bash <(curl -s "http://${HYDRA_BOOT}/render/start-${procname}?cluster=${CLUSTER_ID}&hostname=${CLUSTER_HOST}")
+		done
 	done
 }
 
 function stopProcessList() {
 	for proc in $*; do
-		echo -n "fetching ${proc} stop... "
-		bash <(curl -s "http://${HYDRA_BOOT}/render/stop-${proc}?cluster=${CLUSTER_ID}&hostname=${CLUSTER_HOST}")
+		echo "${proc}" | tr ':' ' ' | while read procname procarg; do
+			echo -n "fetching ${procname} stop... "
+			bash <(curl -s "http://${HYDRA_BOOT}/render/stop-${procname}?cluster=${CLUSTER_ID}&hostname=${CLUSTER_HOST}")
+		done
 	done
 	return 0
 }
@@ -52,6 +57,11 @@ case $1 in
 		exec bash <(curl -s "http://${HYDRA_BOOT}/render/task?cluster=${CLUSTER_ID}&hostname=${CLUSTER_HOST}") $*
 		exit
 		;;
+	task-clone)
+		shift
+		exec bash <(curl -s "http://${HYDRA_BOOT}/render/task-clone?cluster=${CLUSTER_ID}&hostname=${CLUSTER_HOST}") $*
+		exit
+		;;
 esac
 
 case "$1-$2" in

diff --git a/config/template/start-minion b/config/template/start-minion
@@ -6,5 +6,7 @@ export broker=$(get_broker)
 export zookeeper=$(get_zookeeper)
 export spawn=$(wait_nodes spawn)
 
+[ ! -z "${PROC_ARG}" ] && export type="-Dminion.type=${PROC_ARG}"
+
 register_node minion
-startProcess minion ${HYDRA_CMD} ${broker} ${zookeeper} {{ minion.cmdline }} ${HYDRA_EXE} minion minion
+startProcess minion ${HYDRA_CMD} ${broker} ${zookeeper} {{ minion.cmdline }} ${type} ${HYDRA_EXE} minion minion
diff --git a/config/template/task-clone b/config/template/task-clone
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+{% include "proc-stub" %}
+
+echo "-----(( task-clone ))-----(( $* ))-----(( $(pwd) ))-----"
+
+nodex=$(which node)
+[ -z "${nodex}" ] && echo "missing node" && exit 1
+
+conf=$1
+nodes=$2
+nodenum=$3
+
+[ ! -f "${conf}" ] && echo "missing config : ${conf}" && exit 1
+[ -z ${nodes} ] && echo "missing node count" && exit 1
+[ -z ${nodenum} ] && echo "missing node number" && exit 1
+
+cat > job.node.js << EOF
+var fs = require('fs');
+var cfg = eval('('+fs.readFileSync('${conf}')+')');
+if (!cfg.jobid) {
+	console.log('echo missing source job id; exit 1');
+	process.exit(1)
+}
+if (!cfg.hosts) {
+	console.log('echo missing source host list; exit 1');
+	process.exit(1)
+}
+var minionPath = cfg.minionPath || 'hydra/minion';
+var dataPath = cfg.dataPath || 'data';
+for (var i in cfg.hosts) {
+	console.log(['rsync -av --delete ',cfg.hosts[i],':',minionPath,'/',cfg.jobid,'/${nodenum}/gold/',dataPath,'/ ',dataPath,'/'].join(''));
+}
+process.exit()
+EOF
+
+${nodex} job.node.js > job.bash
+source job.bash
+exit 0
diff --git a/main.js b/main.js
@@ -14,7 +14,8 @@ var template = {};
 var waiters = {};
 var clusterDefault = {};
 var params = {
-    boothost : [os.hostname(),8007].join(":")
+    boothost : [os.hostname(),8007].join(":"),
+    debug: false
 };
 
 if (!String.prototype.endsWith) {
@@ -210,7 +211,7 @@ var prefix = {
                 if (cluster.shortenHost) {
                     q.hostname = shortenHost(q.hostname);
                 }
-                var host = cluster.node[q.hostname] || cluster.node.defaults;
+                var host = findHostDefinition(cluster, q.hostname);//cluster.node[q.hostname] || cluster.node.defaults;
                 call(res, q, cluster, host);
             });
         }
@@ -289,6 +290,22 @@ var defaultImageRoot = function() {
     return ['http://',params.boothost,'/image/default'].join('')
 };
 
+var findHostDefinition = function(cluster, hostname) {
+    if (cluster.node[hostname]) {
+        debug(['fhd-x',hostname]);
+        return cluster.node[hostname];
+    }
+    for (var key in cluster.node) {
+        var hostDef = cluster.node[key];
+        if (hostDef.hosts && hostDef.hosts.indexOf(hostname) >= 0) {
+        debug(['fhd-h',hostname]);
+            return hostDef;
+        }
+    }
+    debug(['fhd-d',hostname]);
+    return cluster.node.defaults;
+}
+
 var oGet = function(obj, find, dv) {
     var o = obj;
     if (!Array.isArray(find)) {
@@ -333,6 +350,10 @@ var setDefaults = function(oS, oT) {
     }
 };
 
+var debug = function(msg) {
+    if (params.debug) console.log(msg);
+}
+
 var api = {
     wait_nodes : function(query, remote, callback) {
         if (!query.cluster) return callback("missing cluster");