pcp2openmetrics: minor bug fixes, addition of archive timestamps

Improvements to pcp2openmetrics handling of labels and  -F output to file option.
Add the PCP5 metrics semantics line consistent with pmproxy.
Also added timestamps to output in archive mode conforming to
the OpenMetrics spec. Existing QA tests have been updated to
verify these changes.

Fixes Red Hat issues RHEL -32434 and RHEL -32407

qa/1131.out

@@ -2270,6 +2270,7 @@ QA output created by 1131
 
 
 # HELP hinv_ncpu number of CPUs in the system
+# PCP5 hinv_ncpu 60.0.32 u32 PM_INDOM_NULL discrete
 # TYPE hinv_ncpu gauge
 hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} NCPU
 ---

qa/1827.out

@@ -7,7 +7,8 @@ pcp2openmetrics HTTP POST (sorted):
 
 
 
-# HELP hinv_ncpu number of CPUs in the system
+# HELP hinv_ncpu b'number of CPUs in the system'
+# PCP5 hinv_ncpu 60.0.32 u32 PM_INDOM_NULL discrete
 # TYPE hinv_ncpu gauge
 Accept: */*
 Body:

src/pcp2openmetrics/pcp2openmetrics.py

@@ -398,24 +398,25 @@ def write_header(self):
         """ Write info header """
         output = self.outfile if self.outfile else "stdout"
         if self.context.type == PM_CONTEXT_ARCHIVE:
-            sys.stdout.write('{ "//": "Writing %d archived metrics to %s..." }\n{ "//": "(Ctrl-C to stop)" }\n' % (len(self.metrics), output))
+            sys.stdout.write('"# Writing %d archived metrics to %s..." }\n{ "//": "(Ctrl-C to stop)" }\n' % (len(self.metrics), output))
             return
 
-        sys.stdout.write('{ "//": "Waiting for %d metrics to be written to %s' % (len(self.metrics), output))
+        sys.stdout.write('# "Waiting for %d metrics to be written to %s' % (len(self.metrics), output))
         if self.runtime != -1:
-            sys.stdout.write(':" }\n{ "//": "%s samples(s) with %.1f sec interval ~ %d sec runtime." }\n' % (self.samples, float(self.interval), self.runtime))
+            sys.stdout.write('\n # "%s samples(s) with %.1f sec interval ~ %d sec runtime." }\n' % (self.samples, float(self.interval), self.runtime))
         elif self.samples:
             duration = (self.samples - 1) * float(self.interval)
-            sys.stdout.write(':" }\n{ "//": "%s samples(s) with %.1f sec interval ~ %d sec runtime." }\n' % (self.samples, float(self.interval), duration))
+            sys.stdout.write('\n # "%s samples(s) with %.1f sec interval ~ %d sec runtime." }\n' % (self.samples, float(self.interval), duration))
         else:
-            sys.stdout.write('..." }\n{ "//": "(Ctrl-C to stop)" }\n')
+            sys.stdout.write('...\n# "(Ctrl-C to stop)" }\n')
 
     def write_openmetrics(self, timestamp):
         """ Write results in openmetrics format """
         if timestamp is None:
             # Silent goodbye, close in finalize()
             return
 
+        self.context.pmNewZone("UTC")
         ts = self.context.datetime_to_secs(self.pmfg_ts(), PM_TIME_SEC)
 
         if self.prev_ts is None:
@@ -430,17 +431,17 @@ def write_openmetrics(self, timestamp):
         def get_type_string(desc):
             """ Get metric type as string """
             if desc.contents.type == pmapi.c_api.PM_TYPE_32:
-                mtype = "32-bit signed"
+                mtype = "32"
             elif desc.contents.type == pmapi.c_api.PM_TYPE_U32:
-                mtype = "32-bit unsigned"
+                mtype = "u32"
             elif desc.contents.type == pmapi.c_api.PM_TYPE_64:
-                mtype = "64-bit signed"
+                mtype = "64"
             elif desc.contents.type == pmapi.c_api.PM_TYPE_U64:
-                mtype = "64-bit unsigned"
+                mtype = "u64"
             elif desc.contents.type == pmapi.c_api.PM_TYPE_FLOAT:
-                mtype = "32-bit float"
+                mtype = "float"
             elif desc.contents.type == pmapi.c_api.PM_TYPE_DOUBLE:
-                mtype = "64-bit float"
+                mtype = "float"
             elif desc.contents.type == pmapi.c_api.PM_TYPE_STRING:
                 mtype = "string"
             else:
@@ -465,6 +466,9 @@ def openmetrics_labels(inst, name, desc, labels):
             new_dict = {}
             new_dict['semantics'] = self.context.pmSemStr(desc.contents.sem)
             new_dict['type'] = get_type_string(desc)
+            if desc.indom != PM_INDOM_NULL:
+                new_dict['instname'] = name
+                new_dict['instid'] = inst
             for key in labels:
                 new_dict.update(labels[key])
             if self.everything:
@@ -483,14 +487,22 @@ def openmetrics_labels(inst, name, desc, labels):
 
         body = ''
         for metric in results:
+            #variable declaration
             i = list(self.metrics.keys()).index(metric)
             desc = self.pmconfig.descs[i]
             context = self.pmfg.get_context()
             pmid = context.pmLookupName(metric)
             labels = context.pmLookupLabels(pmid[0])
+            semantics = self.context.pmSemStr(desc.contents.sem)
+            units = desc.contents.units
+            pmIDStr = context.pmIDStr(pmid[0])
+            pmIndomStr = context.pmInDomStr(desc)
             help_dict = {}
             help_dict[metric] = context.pmLookupText(pmid[0])
 
+            if self.context.type == PM_CONTEXT_ARCHIVE:
+                body += "\nMetric %s details (last fetch: %d)\n:" % (metric, ts)
+            body += '# PCP5 %s %s %s %s %s %s\n' % (openmetrics_name(metric), pmIDStr, get_type_string(desc), pmIndomStr, semantics, units)
             body += '# HELP %s %s\n' % (openmetrics_name(metric), help_dict[metric])
             body += '# TYPE %s %s\n' % (openmetrics_name(metric), openmetrics_type(desc))
 
@@ -503,7 +515,10 @@ def openmetrics_labels(inst, name, desc, labels):
                     value = format(value, fmt)
                 else:
                     str(value)
-                body += '%s%s %s\n' % (openmetrics_name(metric), openmetrics_labels(inst, name, desc, labels), value)
+                if self.context.type == PM_CONTEXT_ARCHIVE:
+                    body += '%s%s %s %s\n' % (openmetrics_name(metric), openmetrics_labels(inst, name, desc, labels), value, ts)
+                else:
+                    body += '%s%s %s\n' % (openmetrics_name(metric), openmetrics_labels(inst, name, desc, labels), value)
 
         if self.url:
             auth = None
@@ -519,6 +534,8 @@ def openmetrics_labels(inst, name, desc, labels):
             except requests.exceptions.ConnectionError as post_error:
                 msg = "Cannot connect to server at %s: %s\n" % (self.url, str(post_error))
                 sys.stderr.write(msg)
+        elif self.outfile:
+            self.writer.write(body)
         else:
             print(body)