Have update-all-remote-wn-ce run updaters with a timeout (SOFTWARE-4081)

Author: matyasselmeci

config/endpoints.ini

@@ -15,10 +15,12 @@
 ;;remote_dir = /home/bosco/osg-wn-client
 ;;; ssh_key (optional) is the SSH key to use to log in with
 ;;ssh_key = /home/osg/.ssh/id_rsa_bosco
-
+;;; timeout (optional, default 3600 i.e. 1 hour) is the number of seconds to give the updater before killing it
+;;timeout = 3600
 
 ;;; These are default values that will be used unless overridden in the other
 ;;; sections.
 [DEFAULT]
 upstream_url = https://repo.opensciencegrid.org/tarball-install/3.5/osg-wn-client-latest.el7.x86_64.tar.gz
 remote_dir = /home/bosco/osg-wn-client
+timeout = 3600

scripts/update-all-remote-wn-clients

@@ -9,6 +9,7 @@ import re
 from subprocess import Popen, STDOUT
 import sys
 
+# noinspection PyUnresolvedReferences
 from six.moves import configparser
 
 
@@ -46,6 +47,18 @@ def call_updater(cfg, section, updater_script, log_dir, dry_run=False):
     upstream_url = safe_get("upstream_url")
     remote_user = safe_get("remote_user")
     remote_dir = safe_get("remote_dir")
+    # Have default timeout be nonzero
+    if not cfg.has_section(section) or not cfg.has_option(section, "timeout"):
+        timeout = "3600"
+    else:
+        timeout = safe_get("timeout")
+    try:
+        if timeout:
+            timeout = int(timeout)
+        else:
+            timeout = 0
+    except ValueError as e:
+        raise Error("%s: Invalid timeout %s" % (section, timeout))
 
     if not local_user:
         raise Error("%s: Missing required option local_user" % section)
@@ -64,6 +77,10 @@ def call_updater(cfg, section, updater_script, log_dir, dry_run=False):
     if remote_dir:
         cmd.append("--remote-dir=%s" % remote_dir)
 
+    if timeout:
+        cmd.insert(0, "timeout")
+        cmd.insert(1, str(timeout))
+
     if dry_run:
         print("Would run %r as %s" % (cmd, local_user))
         return
@@ -86,6 +103,11 @@ def call_updater(cfg, section, updater_script, log_dir, dry_run=False):
 
     if returncode == 0:
         log.info("Endpoint %s ok.", endpoint)
+    elif returncode == 124:
+        raise Error(
+            "Endpoint %s TIMED OUT.  See %s for details."
+            % (endpoint, log_file_path)
+        )
     else:
         raise Error(
             "Endpoint %s FAILED with error %d.  See %s for details."