Merge pull request #117 from HPC-SimTools/new_portal

Get new portal working

Author: rosswhitfield

ipsframework/configurationManager.py

@@ -217,17 +217,11 @@ def initialize(self, data_mgr, resource_mgr, task_mgr):
         else:
             use_accurate_nodes = True
 
-        user_def_tprocs = int(self.platform_conf.get('TOTAL_PROCS', 0))
-        user_def_nodes = int(self.platform_conf.get('NODES', 0))
-        user_def_ppn = int(self.platform_conf.get('PROCS_PER_NODE', 0))
-        user_def_cpn = int(self.platform_conf.get('CORES_PER_NODE', 0))
-        user_def_spn = int(self.platform_conf.get('SOCKETS_PER_NODE', 0))
-
-        self.platform_conf['TOTAL_PROCS'] = user_def_tprocs
-        self.platform_conf['NODES'] = user_def_nodes
-        self.platform_conf['PROCS_PER_NODE'] = user_def_ppn
-        self.platform_conf['CORES_PER_NODE'] = user_def_cpn
-        self.platform_conf['SOCKETS_PER_NODE'] = user_def_spn
+        self.platform_conf['TOTAL_PROCS'] = int(self.platform_conf.get('TOTAL_PROCS', 0))
+        self.platform_conf['NODES'] = int(self.platform_conf.get('NODES', 0))
+        self.platform_conf['PROCS_PER_NODE'] = int(self.platform_conf.get('PROCS_PER_NODE', 0))
+        self.platform_conf['CORES_PER_NODE'] = int(self.platform_conf.get('CORES_PER_NODE', 0))
+        self.platform_conf['SOCKETS_PER_NODE'] = int(self.platform_conf.get('SOCKETS_PER_NODE', 0))
         self.platform_conf['USE_ACCURATE_NODES'] = use_accurate_nodes
         self.platform_conf['MPIRUN_VERSION'] = mpirun_version
 
@@ -253,6 +247,10 @@ def initialize(self, data_mgr, resource_mgr, task_mgr):
                     if key not in conf_keys:
                         conf[key] = self.platform_conf[key]
 
+                # Override platform value for PORTAL_URL if in simulation
+                if 'PORTAL_URL' in conf_keys:
+                    self.platform_conf['PORTAL_URL'] = conf['PORTAL_URL']
+
             except (IOError, SyntaxError):
                 self.fwk.exception('Error opening config file %s: ', conf_file)
                 raise
@@ -382,20 +380,14 @@ def _initialize_fwk_components(self):
                 portal_conf['USER'] = self.sim_map[self.fwk_sim_name].sim_conf['USER']
             except KeyError:
                 portal_conf['USER'] = self.platform_conf['USER']
-            havePortal = True
             if self.fwk.log_level == logging.DEBUG:
                 portal_conf['LOG_LEVEL'] = 'DEBUG'
 
-            try:
-                portal_conf['PORTAL_URL'] = self.get_platform_parameter('PORTAL_URL', silent=True)
-                portal_conf['RUNID_URL'] = self.get_platform_parameter('RUNID_URL', silent=True)
-            except KeyError:
-                havePortal = False
+            portal_conf['PORTAL_URL'] = self.get_platform_parameter('PORTAL_URL', silent=True)
 
-            if havePortal:
-                component_id = self._create_component(portal_conf,
-                                                      self.sim_map[self.fwk_sim_name])
-                self.fwk_components.append(component_id)
+            component_id = self._create_component(portal_conf,
+                                                  self.sim_map[self.fwk_sim_name])
+            self.fwk_components.append(component_id)
 
     def _initialize_sim(self, sim_data):
         """

ipsframework/portalBridge.py

@@ -5,9 +5,6 @@
 
 import datetime
 import sys
-import urllib.request
-import urllib.parse
-import urllib.error
 import os
 from subprocess import Popen, PIPE
 import time
@@ -17,6 +14,7 @@
 import glob
 import itertools
 import json
+import shutil
 from ipsframework import ipsutil, Component
 from ipsframework.convert_log_function import convert_logdata_to_html
 
@@ -88,7 +86,6 @@ def __init__(self, services, config):
         self.startTime = self.curTime
         self.services = services
         self.sim_map = {}
-        self.runid_url = None
         self.portal_url = None
         self.done = False
         self.first_event = True
@@ -109,14 +106,10 @@ def init(self, timestamp=0.0, **keywords):
         Try to connect to the portal, subscribe to *_IPS_MONITOR* events and
         register callback :py:meth:`.process_event`.
         """
-        # try:
-        #     self.portal_url = self.PORTAL_URL
-        # except AttributeError:
-        #     pass
-        # try:
-        #     self.runid_url = self.RUNID_URL
-        # except AttributeError:
-        #     pass
+        try:
+            self.portal_url = self.PORTAL_URL
+        except AttributeError:
+            pass
         self.host = self.services.get_config_param('HOST')
         self.services.subscribe('_IPS_MONITOR', "process_event")
         try:
@@ -268,16 +261,17 @@ def send_event(self, sim_data, event_data):
                     self.services.exception("Error writing html file into USER_W3_DIR directory")
                     self.write_to_htmldir = False
         if self.portal_url:
-            webmsg = urllib.parse.urlencode(event_data).encode("utf-8")
+            webmsg = json.dumps(event_data)
             try:
                 if self.first_event:  # First time, launch sendPost.py daemon
-                    cmd = os.path.join(sys.path[0], 'sendPost.py')
-                    self.childProcess = Popen(cmd, shell=True, bufsize=128,
+                    cmd = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sendPost.py')
+                    python_exec = shutil.which('python')
+                    self.childProcess = Popen([python_exec, cmd], bufsize=128,
                                               stdin=PIPE, stdout=PIPE,
                                               stderr=PIPE, close_fds=True)
                     self.first_event = False
-                self.childProcess.stdin.write('%s %s\n' %
-                                              (self.portal_url, webmsg))
+                self.childProcess.stdin.write(('%s %s\n' %
+                                               (self.portal_url, webmsg)).encode())
                 self.childProcess.stdin.flush()
             except Exception as e:
                 self.services.exception('Error transmitting event number %6d to %s : %s',
@@ -481,15 +475,6 @@ def init_simulation(self, sim_name, sim_root):
         d = datetime.datetime.now()
         date_str = "%s.%03d" % (d.strftime("%Y-%m-%dT%H:%M:%S"), int(d.microsecond / 1000))
         sim_data.portal_runid = "_".join([self.host, "USER", date_str])
-        if self.runid_url is not None:
-            self.services.debug('PORTAL_RUNID_URL = %s', str(self.runid_url))
-            try:
-                f = urllib.request.urlopen(self.runid_url, None, 10)
-                sim_data.portal_runid = f.read().strip()
-            except (urllib.error.URLError) as e:
-                self.services.error('Error obtaining runID from service at %s : %s' %
-                                    (self.runid_url, str(e)))
-                self.services.error('Using a datetime instead')
         try:
             self.services.set_config_param('PORTAL_RUNID', sim_data.portal_runid,
                                            target_sim_name=sim_name)

ipsframework/sendPost.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# -------------------------------------------------------------------------------
+#  Copyright 2006-2020 UT-Battelle, LLC. See LICENSE for more information.
+# -------------------------------------------------------------------------------
+
+import sys
+from urllib import request, error
+import socket
+import time
+import traceback
+
+headers = {'Content-Type': 'application/json'}
+
+
+def sendEncodedMessage(url, msg):
+    if not isinstance(msg, bytes):
+        msg = msg.encode('utf-8')
+
+    num_trials = 2
+    trial = 0
+    delay = [0.4, 0.8, 1.2]
+
+    while trial < num_trials:
+        try:
+            req = request.Request(url, data=msg, headers=headers, method='POST')
+            resp = request.urlopen(req)
+        except error.URLError:
+            trial += 1
+            if trial > num_trials:
+                open('PORTAL.err', 'a').write('%s\n' % (msg))
+            else:
+                time.sleep(delay[trial-1])
+        else:
+            break
+    try:
+        resp.close()
+    except Exception:
+        pass
+
+
+if __name__ == "__main__":
+    """ Loop over input from stdin, expecting lines of the format:
+        URL ENCODED_WEB_MSG
+    """
+    timeout = 3
+    socket.setdefaulttimeout(timeout)
+    error_f = open("sendpost.err", 'w')
+    line = '   '
+    while True:
+        try:
+            line = sys.stdin.readline().rstrip('\n')
+            if line == '':
+                break
+            tokens = line.split(' ', 1)
+            sendEncodedMessage(tokens[0], tokens[1])
+        except Exception:
+            traceback.print_exc(file=error_f)
+    sys.exit(0)

setup.cfg

@@ -1,6 +1,6 @@
 [flake8]
 max-line-length = 160
-max-complexity = 38
+max-complexity = 39
 exclude =
     ipsframework/configobj.py,
     ipsframework/six.py,