fix bugs exposed by py3 port testing

ldms/scripts/ldms-csv-anonymize

@@ -567,14 +567,14 @@ mapmagic = "#anonymize-csv-map"
 def write_map(maps, kind, fn):
     if len(maps[kind]) > 0:
         with open(fn, "w") as o:
-            print(mapmagic, kind)
+            print(mapmagic, kind, file=o)
             m = maps[kind]
             for key in sorted(m):
                 if key == "netdomains":
                     if len(m[key]) > 0:
-                        print(key, ",".join(m[key]))
+                        print(key, ",".join(m[key]), file=o)
                 else:
-                    print(key, m[key])
+                    print(key, m[key], file=o)
 
 def reload_map(m, fn, kind):
     """prime m with data from prior run. input can be empty, otherwise must match kind."""
@@ -696,7 +696,7 @@ if __name__ == "__main__":
         print(files)
         print(cols)
     maps = dict()
-    for key in rewrite.keys():
+    for key in list(rewrite.keys()):
         maps[key] =  dict()
     if args.imap:
         reload_map(maps["int"], args.imap, "int")
@@ -707,15 +707,15 @@ if __name__ == "__main__":
     maps["host"]["netdomains"] = []
     if args.hmap:
         reload_map(maps["host"], args.hmap, "host")
-        if maps["host"].has_key("netdomains"):
+        if "netdomains" in maps["host"]:
             maps["host"]["netdomains"] = maps["host"]["netdomains"].split(",")
 
     prefix = args.save_maps
     if not prefix:
         prefix = 'anonmap_'
     seed = args.seed
     if not seed:
-        seed = random.randint(1,sys.maxint)
+        seed = random.randint(1,sys.maxsize)
     random.seed(seed)
     od = args.out_dir
     if not od or not os.path.isdir(od):
@@ -735,7 +735,7 @@ if __name__ == "__main__":
                 for ln in i:
                     ln = ln.rstrip()
                     if len(ln) < 3:
-                        print(ln)
+                        print(ln, file=o)
                         continue
                     if ln[0] == '#':
                         x = ln.split(sep)
@@ -744,7 +744,7 @@ if __name__ == "__main__":
                             k = convert_col(k)
                             x[k] = map_header(x[k], method)
                         y = sep.join(x)
-                        print(y)
+                        print(y, file=o)
                         continue
                     x = ln.split(sep)
                     for c in cols:
@@ -755,7 +755,7 @@ if __name__ == "__main__":
                         k = convert_col(k)
                         x[k] = rewrite[method](x[k], maps[method])
                     y = sep.join(x)
-                    print(y)
+                    print(y, file=o)
 
 
     # print seed file if we ever use random

ldms/scripts/ldms-csv-export-sos

@@ -66,7 +66,7 @@ ldms_type_sos_map = { "char" : "uint16",
                       "timestamp" : "timestamp"
     }
 
-match_elements = lambda p, s: [i for (q, i) in zip(s, range(len(s))) if p == q]
+match_elements = lambda p, s: [i for (q, i) in zip(s, list(range(len(s)))) if p == q]
 
 def fileopener(fn):
     """return the correct open function for the filename (default or gzip.open)"""
@@ -205,7 +205,7 @@ def value_type(v):
     if len(v) == 0:
             return "u64"
     try:
-        x = long(v)
+        x = int(v)
         if x < 0:
             if x < -9223372036854775808:
                 # cannot happen on s64 data, so must be
@@ -506,8 +506,7 @@ def format_schema_prolog(f, args):
     s = """{
     "name" : """ + "\"" + args.dataname + """",
     "attrs" : ["""
-    with open(f, "w") as f:
-        print(s)
+    print(s, file=f)
 
 def format_join(f, j, col_heads):
     jname =  j[0]
@@ -522,8 +521,7 @@ def format_join(f, j, col_heads):
             s += ", "
         s += "\"" + i + "\""
     s += "], \"index\" : {} }"
-    with open(f, "w") as f:
-        print(f"{s},")
+    print(s, end=' ', file=f)
 
 default_joins = [
     ("comp_job_time", "component_id", "job_id", "Time"),
@@ -543,8 +541,7 @@ def format_schema_epilog(f, args, col_heads):
     s="\n"
     s += """  ]
 }"""
-    with open(f, "w") as f:
-        print(s)
+    print(s, file=f)
 
 def is_index(n, args):
     if n in ["Time", "component_id", "job_id", "component_id.value", "job_id.value", "ProducerName", "anonymized_host_ProducerName"]:
@@ -573,16 +570,13 @@ def format_schema_element(f, args, name, mtype, first):
     if is_index(name, args):
         s += ",\t\"index\" : {} "
     s += " }"
-    with open(f, "w") as f:
-        print(f"{s},")
+    print(s, end=' ', file=f)
 
 def format_map_prolog(f):
-    with open(f, "w") as f:
-        print(f"[")
+    print("[", file=f)
 
 def format_map_epilog(f):
-    with open(f, "w") as f:
-        print(f"\n]")
+    print("\n]", file=f)
 
 def format_map_element(f, k, metric, first):
     # accepts k as in or array of ints
@@ -595,8 +589,7 @@ def format_map_element(f, k, metric, first):
     else:
         colspec = "\"list\" : [" + str(k).strip("[]") +  "]"
     s += "\t{ \"target\" : \"" + metric + "\", \"source\" : { " + colspec + " }}"
-    with open(f, "w") as f:
-        print(f"{s},")
+    print(s, end=' ', file=f)
 
 def generate_map_new(col_heads, col_kinds, arr, udata, mapout, sout, mapvec):
     """ generate map conforming to schema and input file. This skips type checking."""

ldms/scripts/ldms_local_opa2test.sh.in

@@ -4,7 +4,7 @@ prefix=@prefix@
 exec_prefix=@exec_prefix@
 libdir=@libdir@
 
-export PYTHONPATH=${prefix}/lib/python2.7/site-packages
+export PYTHONPATH=@pythondir@
 . $libdir/ovis-lib-configvars.sh
 . $libdir/ovis-ldms-configvars.sh
 # Manually redefine portbase to avoid collisions.

ldms/src/sampler/filesingle/ldms-sensors-config

@@ -1,18 +1,14 @@
 #! /usr/bin/env python3
+# usage: $0: sensors_bin_path
 # Run sensors under strace to discover where sensor files
 # live on the current system and generate a draft file.
-
-import argparse
-import os
-import subprocess
-import tempfile
-
 import warnings
 import sys
 import os.path
 import tempfile
 import subprocess
 import argparse
+import os
 
 def fix_label(s, nodash=False):
     if s[-2:] == "\\n":
@@ -86,7 +82,7 @@ def main():
 
     devices = { }
     with open(trace_file) as file:
-        lines = f.readlines()
+        lines = file.readlines()
 
     if not args.test_sensors:
         os.unlink(trace_file)
@@ -147,9 +143,9 @@ def main():
                 base_name = os.path.basename(last_file)
                 split_index = base_name.rfind("_")
                 item = base_name[:split_index]
-                if "group" not in devices[last_device]:
+                if not "group" in devices[last_device]:
                     devices[last_device]["group"] = last_group
-                if item not in devices[last_device]["items"]:
+                if not item in devices[last_device]["items"]:
                     devices[last_device]["items"].append(item)
                     devices[last_device][item] = {}
                     devices[last_device][item]["label"] = fix_label(temp)
@@ -159,9 +155,9 @@ def main():
                 base_name = os.path.basename(last_file)
                 split_index = base_name.rfind("_")
                 item = base_name[:split_index]
-                if "group" not in devices[last_device]:
+                if not "group" in devices[last_device]:
                     devices[last_device]["group"] = last_group
-                if item not in devices[last_device]["items"]:
+                if not item in devices[last_device]["items"]:
                     devices[last_device]["items"].append(item)
                     devices[last_device]["group"] = last_group
                     devices[last_device][item] = {}
@@ -171,13 +167,15 @@ def main():
             if last_file_type in IGNORE_TYPES:
                 continue
             continue
-        if ":" not in line:
+        # exclude   lines with "xxxxx:" and "sdf: bar" but allow : in mlx device names.
+        if ": " not in line and line.strip()[-1] != ':':
             last_group = line
 
     for device, data in devices.items():
         if data["items"]:
             for item in data["items"]:
-                item_label = fix_label(".".join([data["group"], data[item]["label"] ]), nodash)
+                xg = data["group"] if data["group"] else "NOGROUP"
+                item_label = fix_label(".".join([xg, data[item]["label"] ]), nodash)
                 print(f"{item_label} {data[item]['inputfile']} S64 -1")
 
     devices = {}

ldms/src/sampler/shm/mpi_profiler/mpi_profiler.c

@@ -313,8 +313,9 @@ void post_Pcontrol_call(const int flag)
 	}
 }
 
-void *ldms_shm_periodic_update()
+void *ldms_shm_periodic_update(void *ignored)
 {
+	(void)ignored;
 	if(log_level_info())
 		printf("INFO: ldms_shm_updater_thread (%d): starting!\n\r",
 				profiler->rankid);

ldms/src/sampler/shm/mpi_profiler/wrap/wrap.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #################################################################################################
 # Copyright (c) 2010, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
@@ -30,7 +30,7 @@
 # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #################################################################################################
-from __future__ import print_function
+
 usage_string = \
 '''Usage: wrap.py [-fgd] [-i pmpi_init] [-c mpicc_name] [-o file] wrapper.w [...]
  Python script for creating PMPI wrappers. Roughly follows the syntax of
@@ -105,6 +105,7 @@
 #include <mpi.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include "userheader.h"
 
 #ifndef _EXTERN_C_
 #ifdef __cplusplus
@@ -237,7 +238,7 @@ def joinlines(list, sep="\n"):
         return ""
 
 # Possible types of Tokens in input.
-LBRACE, RBRACE, TEXT, IDENTIFIER = range(4)
+LBRACE, RBRACE, TEXT, IDENTIFIER = list(range(4))
 
 class Token:
     """Represents tokens; generated from input by lexer and fed to parse()."""
@@ -481,7 +482,7 @@ def types(self):
         return [arg.cType() for arg in self.args]
 
     def argsNoEllipsis(self):
-        return filter(lambda arg: arg.name != "...", self.args)
+        return [arg for arg in self.args if arg.name != "..."]
 
     def returnsErrorCode(self):
         """This is a special case for MPI_Wtime and MPI_Wtick.
@@ -578,7 +579,7 @@ def enumerate_mpi_declarations(mpicc, includes):
                 raise ValueError("Malformed declaration in header: '%s'" % line)
 
             arg_string = line[lparen+1:rparen]
-            arg_list = list(map(lambda s: s.strip(), arg_string.split(",")))
+            arg_list = list([s.strip() for s in arg_string.split(",")])
 
             # Handle functions that take no args specially
             if arg_list == ['void']:
@@ -659,7 +660,7 @@ def write_fortran_binding(out, decl, delegate_name, binding, stmts=None):
     out.write(decl.fortranPrototype(binding, default_modifiers))
     out.write(" { \n")
     if stmts:
-        out.write(joinlines(map(lambda s: "    " + s, stmts)))
+        out.write(joinlines(["    " + s for s in stmts]))
     if decl.returnsErrorCode():
         # regular MPI fortran functions use an error code
         out.write("    %s(%s);\n" % (delegate_name, ", ".join(decl.fortranArgNames())))
@@ -1092,7 +1093,7 @@ def __init__(self):
 
     def iwrite(self, file, level, text):
         """Write indented text."""
-        for x in xrange(level):
+        for x in range(level):
             file.write("  ")
         file.write(text)
 

ldms/src/sampler/shm/shm_sampler.c

@@ -366,9 +366,9 @@ static int create_base_for_box(ldms_shm_box_t *box, const char* schema_name,
 	box->base->job_end_idx = initial_base_config->job_end_idx;
 	box->base->job_id_idx = initial_base_config->job_id_idx;
 	box->base->job_start_idx = initial_base_config->job_start_idx;
-	box->base->uid = initial_base_config->uid;
-	box->base->gid = initial_base_config->gid;
-	box->base->perm = initial_base_config->perm;
+	box->base->auth.uid = initial_base_config->auth.uid;
+	box->base->auth.gid = initial_base_config->auth.gid;
+	box->base->auth.perm = initial_base_config->auth.perm;
 
 	box->base->producer_name = strdup(initial_base_config->producer_name);
 	box->base->schema_name = strdup(schema_name);
@@ -439,14 +439,14 @@ static void handle_ldms_set_delete_issue(ldms_shm_box_t* box)
 								ldms_shm_metric_set_counter)
 								+ 1;
 		int num_delimiter_chars = 1;
-		int instance_name_len = strlen(box->base->instance_name)
+		size_t instance_name_len = strlen(box->base->instance_name)
 				+ num_delimiter_chars
 				+ ldms_shm_metric_set_counter_len + 1;
 
 		char* instance_name = calloc(instance_name_len, sizeof(char));
 		if(!instance_name) {
 			ovis_log(mylog, OVIS_LERROR,
-					"failed to allocate memory of size=%d (%d + %d + %d + 1) for instance %s\n",
+					"failed to allocate memory of size=%zu (%zu + %d + %d + 1) for instance %s\n",
 					instance_name_len,
 					strlen(box->base->instance_name),
 					num_delimiter_chars,
@@ -482,7 +482,7 @@ static int create_metric_set(ldms_shm_box_t *box, ldms_shm_set_t shm_set,
 	if(!box->base->schema) {
 		ovis_log(mylog, OVIS_LERROR,
 				"%s: The schema '%s' could not be created, errno=%d.\n",
-				__FILE__, box->base, errno);
+				__FILE__, box->base->schema_name, errno);
 		rc = errno;
 		return rc;
 	}