Updated pcp2openmetrics tool to fix issues: RHEL-39159, RHEL-39132, R…

…HEL-32983, RHEL-39293. Added command line option to exclude # PCP 5 comment line Updated output to follow openmetrics format to include _total metric name ending Omitted blank line after each metric output to adhere to prometheus openmetrics parser Added # EOF line to end of text dump to adhere to prometheus openmetrics parser Updated relevent qa for pcp2openmetric changes (1131 & 1827) Updated pcp2openmetrics man page and fixed style errors(2)
performancecopilot · Jul 3, 2024 · 6cefac3 · 6cefac3
1 parent c527872
commit 6cefac3
Show file tree

Hide file tree

Showing 6 changed files with 77 additions and 58 deletions.
diff --git a/qa/1131 b/qa/1131
@@ -62,9 +62,10 @@ pcp2json -a $A -H -I -Z UTC+0 -x "" | _archive_filter
 echo "---"
 pcp2json -a $A -H -I -z -X -b GB -P 2 -F $tmp.outfile ""
 echo "---"
-pcp2openmetrics -s1 -H -z hinv.ncpu | _filter_pcp2openmetrics
+pcp2openmetrics -s1 -z hinv.ncpu | _filter_pcp2openmetrics
+echo "---"
+pcp2openmetrics -s2 -x hinv.ncpu | _filter_pcp2openmetrics
 echo "---"
-
 
 cat $tmp.outfile | _archive_filter
 which json_verify > /dev/null 2>&1

diff --git a/qa/1131.out b/qa/1131.out
@@ -2267,13 +2267,18 @@ QA output created by 1131
 }
 ---
 ---
-
-
+# EOF
 # HELP hinv_ncpu number of CPUs in the system
 # PCP5 hinv_ncpu 60.0.32 u32 PM_INDOM_NULL discrete
 # TYPE hinv_ncpu gauge
 hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} NCPU
 ---
+# EOF
+# HELP hinv_ncpu number of CPUs in the system
+# TYPE hinv_ncpu gauge
+hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} NCPU
+hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} NCPU
+---
 {
   "@pcp": {
     "@hosts": [

diff --git a/qa/1827 b/qa/1827
@@ -63,12 +63,20 @@ $PCP_PYTHON_PROG $here/src/pythonserver.py $port >$tmp.python.out 2>&1 &
 pid=$!
 sleep 2 # let server start up
 
+echo "---" 
 echo "pcp2openmetrics invocation" | tee -a $here/$seq.full
 pcp2openmetrics -s1 -u http://localhost:$port hinv.ncpu >$tmp.openmetrics.out 2>$tmp.openmetrics.err
 
 echo "pcp2openmetrics HTTP POST (sorted):"
 _filter_pcp2openmetrics_http <$tmp.python.out
 
+cat /dev/null > $tmp.python.out
+
+echo "---"
+echo "testing -x option" | tee -a $here/$seq.full
+pcp2openmetrics -s1 -x -u http://localhost:$port hinv.ncpu >$tmp.openmetrics2.out 2>$tmp.openmetrics2.err
+_filter_pcp2openmetrics_http <$tmp.python.out
+
 # terminate pythonserver.py now
 pmsignal $pid >/dev/null 2>&1 
 

diff --git a/qa/1827.out b/qa/1827.out
@@ -1,4 +1,5 @@
 QA output created by 1827
+---
 pcp2openmetrics invocation
 pcp2openmetrics HTTP POST (sorted):
 
@@ -21,3 +22,22 @@ INFO:root:Starting httpd...
 Path: /
 User-Agent: python-requests VERSION
 hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} N (cpus)
+---
+testing -x option
+
+
+
+
+
+# HELP hinv_ncpu number of CPUs in the system
+# TYPE hinv_ncpu gauge
+Accept: */*
+Body:
+Content-Length: SIZE
+Content-Type: application/openmetrics-text
+Headers:
+Host: localhost:PORT
+INFO:root:POST request,
+Path: /
+User-Agent: python-requests VERSION
+hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} N (cpus)
diff --git a/src/pcp2openmetrics/pcp2openmetrics.1 b/src/pcp2openmetrics/pcp2openmetrics.1
@@ -181,6 +181,13 @@ Maximum time (in seconds) when sending a HTTP POST.
 Corresponding command line option is \fB\-o\fP.
 Defaults to \fB2.5\fP seconds.
 .RE
+.PP
+no-comment (boolean)
+.RS 4
+Omit # PCP5 comment line. Omits header for some metric
+data such as type, instance domain, and semantics.
+Corresponding command line option is \fB\-x\fP.
+Defaults to \fBno\fP.
 .SH OPTIONS
 The available command line options are:
 .TP 5
@@ -343,9 +350,6 @@ on
 .IR host ,
 rather than from the default localhost.
 .TP
-\fB\-H\fR, \fB\-\-no\-header\fR
-Do not print any headers.
-.TP
 \fB\-i\fR \fIinstances\fR, \fB\-\-instances\fR=\fIinstances\fR
 Retrieve and report only the specified metric
 .IR instances .
@@ -605,8 +609,8 @@ and
 \fB\-V\fR, \fB\-\-version\fR
 Display version number and exit.
 .TP
-\fB\-x\fR, \fB\-\-with\-extended\fR
-Write extended information.
+\fB\-x\fR, \fB\-no\-comment\fR
+Omit # PCP5 comment line
 .TP
 \fB\-X\fR, \fB\-\-with\-everything\fR
 Write everything known about metrics, including PCP internal IDs.

diff --git a/src/pcp2openmetrics/pcp2openmetrics.py b/src/pcp2openmetrics/pcp2openmetrics.py
@@ -50,24 +50,24 @@ def __init__(self):
         self.opts = self.options()
 
         # Configuration directives
-        self.keys = ('source', 'output', 'derived', 'header', 'globals',
-                     'samples', 'interval', 'type', 'precision', 'daemonize',
+        self.keys = ('source', 'output', 'derived', 'globals',
+                     'samples', 'interval', 'precision', 'daemonize',
                      'timefmt', 'everything',
                      'count_scale', 'space_scale', 'time_scale', 'version',
                      'count_scale_force', 'space_scale_force', 'time_scale_force',
-                     'type_prefer', 'precision_force', 'limit_filter', 'limit_filter_force',
+                     'precision_force', 'limit_filter', 'limit_filter_force',
                      'live_filter', 'rank', 'invert_filter', 'predicate', 'names_change',
                      'speclocal', 'instances', 'ignore_incompat', 'ignore_unknown',
                      'omit_flat', 'include_labels', 'url', 'http_user', 'http_pass',
-                     'http_timeout')
+                     'http_timeout', 'no_comment')
 
         # Ignored for pmrep(1) compatibility
         self.keys_ignore = (
-                     'timestamp', 'unitinfo', 'colxrow', 'separate_header', 'fixed_header',
+                     'timestamp','header', 'unitinfo', 'colxrow', 'separate_header', 'fixed_header',
                      'delay', 'width', 'delimiter', 'extcsv', 'width_force',
                      'extheader', 'repeat_header', 'interpol',
                      'dynamic_header', 'overall_rank', 'overall_rank_alt', 'sort_metric',
-                     'instinfo', 'include_texts')
+                     'instinfo', 'include_texts', 'type', 'type_prefer')
 
         # The order of preference for options (as present):
         # 1 - command line options
@@ -79,13 +79,12 @@ def __init__(self):
         self.output = None # For pmrep conf file compat only
         self.speclocal = None
         self.derived = None
-        self.header = 1
         self.globals = 1
         self.samples = None # forever
         self.interval = pmapi.timeval(10)      # 10 sec
         self.opts.pmSetOptionInterval(str(10)) # 10 sec
         self.delay = 0
-        self.type = 0
+        self.type = 1
         self.type_prefer = self.type
         self.ignore_incompat = 0
         self.ignore_unknown = 0
@@ -118,6 +117,8 @@ def __init__(self):
         self.http_user = None
         self.http_pass = None
         self.http_timeout = TIMEOUT
+        self.no_comment = False
+        self.header_flag = True
 
         # Internal
         self.runtime = -1
@@ -146,7 +147,7 @@ def options(self):
         opts = pmapi.pmOptions()
         opts.pmSetOptionCallback(self.option)
         opts.pmSetOverrideCallback(self.option_override)
-        opts.pmSetShortOptions("a:h:LK:c:Ce:D:V?HGA:S:T:O:s:t:rRIi:jJ:4:58:9:nN:vmP:0:q:b:y:Q:B:Y:F:f:Z:zXo:p:U:u:")
+        opts.pmSetShortOptions("a:h:LK:c:Ce:D:V?GA:S:T:O:s:t:Ii:jJ:4:58:9:nN:vmP:0:q:b:y:Q:B:Y:F:f:Z:zXo:p:U:u:x")
         opts.pmSetShortUsage("[option...] metricspec [...]")
 
         opts.pmSetLongOptionHeader("General options")
@@ -166,7 +167,6 @@ def options(self):
         opts.pmSetLongOptionHelp()         # -?/--help
 
         opts.pmSetLongOptionHeader("Reporting options")
-        opts.pmSetLongOption("no-header", 0, "H", "", "omit headers")
         opts.pmSetLongOption("no-globals", 0, "G", "", "omit global metrics")
         opts.pmSetLongOptionAlign()        # -A/--align
         opts.pmSetLongOptionStart()        # -S/--start
@@ -176,8 +176,6 @@ def options(self):
         opts.pmSetLongOptionInterval()     # -t/--interval
         opts.pmSetLongOptionTimeZone()     # -Z/--timezone
         opts.pmSetLongOptionHostZone()     # -z/--hostzone
-        opts.pmSetLongOption("raw", 0, "r", "", "output raw counter values (no rate conversion)")
-        opts.pmSetLongOption("raw-prefer", 0, "R", "", "prefer output raw counter values (no rate conversion)")
         opts.pmSetLongOption("ignore-incompat", 0, "I", "", "ignore incompatible instances (default: abort)")
         opts.pmSetLongOption("ignore-unknown", 0, "5", "", "ignore unknown metrics (default: abort)")
         opts.pmSetLongOption("names-change", 1, "4", "ACTION", "update/ignore/abort on PMNS change (default: ignore)")
@@ -201,6 +199,7 @@ def options(self):
         opts.pmSetLongOption("time-scale-force", 1, "Y", "SCALE", "forced time unit")
 
         opts.pmSetLongOption("with-everything", 0, "X", "", "write everything, incl. internal IDs")
+        opts.pmSetLongOption("no-comment", 0, "x", "", "omit comment lines")
 
         opts.pmSetLongOption("url", 1, "u", "URL", "URL of endpoint to receive HTTP POST")
         opts.pmSetLongOption("http-timeout", 1, "o", "SECONDS", "timeout when sending HTTP POST")
@@ -238,14 +237,8 @@ def option(self, opt, optarg, _index):
                 self.derived = ";" + optarg
             else:
                 self.derived = self.derived + ";" + optarg
-        elif opt == 'H':
-            self.header = 0
         elif opt == 'G':
             self.globals = 0
-        elif opt == 'r':
-            self.type = 1
-        elif opt == 'R':
-            self.type_prefer = 1
         elif opt == 'I':
             self.ignore_incompat = 1
         elif opt == '5':
@@ -298,6 +291,8 @@ def option(self, opt, optarg, _index):
             self.time_scale_force = optarg
         elif opt == 'X':
             self.everything = 1
+        elif opt == 'x':
+            self.no_comment = True
         elif opt == 'u':
             self.url = optarg
         elif opt == 'o':
@@ -319,6 +314,7 @@ def connect(self):
 
         if pmapi.c_api.pmSetContextOptions(self.context.ctx, self.opts.mode, self.opts.delta):
             raise pmapi.pmUsageErr()
+        self.context.pmNewZone("UTC")
 
     def validate_config(self):
         """ Validate configuration options """
@@ -346,11 +342,6 @@ def execute(self):
         # Common preparations
         self.context.prepare_execute(self.opts, False, self.interpol, self.interval)
 
-        # Headers
-        if self.header == 1:
-            self.header = 0
-            self.write_header()
-
         # Just checking
         if self.check == 1:
             return
@@ -394,29 +385,12 @@ def report(self, tstamp):
 
         self.write_openmetrics(tstamp)
 
-    def write_header(self):
-        """ Write info header """
-        output = self.outfile if self.outfile else "stdout"
-        if self.context.type == PM_CONTEXT_ARCHIVE:
-            sys.stdout.write('"# Writing %d archived metrics to %s..." }\n{ "//": "(Ctrl-C to stop)" }\n' % (len(self.metrics), output))
-            return
-
-        sys.stdout.write('# "Waiting for %d metrics to be written to %s' % (len(self.metrics), output))
-        if self.runtime != -1:
-            sys.stdout.write('\n # "%s samples(s) with %.1f sec interval ~ %d sec runtime." }\n' % (self.samples, float(self.interval), self.runtime))
-        elif self.samples:
-            duration = (self.samples - 1) * float(self.interval)
-            sys.stdout.write('\n # "%s samples(s) with %.1f sec interval ~ %d sec runtime." }\n' % (self.samples, float(self.interval), duration))
-        else:
-            sys.stdout.write('...\n# "(Ctrl-C to stop)" }\n')
-
     def write_openmetrics(self, timestamp):
         """ Write results in openmetrics format """
         if timestamp is None:
             # Silent goodbye, close in finalize()
             return
 
-        self.context.pmNewZone("UTC")
         ts = self.context.datetime_to_secs(self.pmfg_ts(), PM_TIME_SEC)
 
         if self.prev_ts is None:
@@ -500,11 +474,12 @@ def openmetrics_labels(inst, name, desc, labels):
             help_dict = {}
             help_dict[metric] = context.pmLookupText(pmid[0])
 
-            if self.context.type == PM_CONTEXT_ARCHIVE:
-                body += "\nMetric %s details (last fetch: %d)\n:" % (metric, ts)
-            body += '# PCP5 %s %s %s %s %s %s\n' % (openmetrics_name(metric), pmIDStr, get_type_string(desc), pmIndomStr, semantics, units)
-            body += '# HELP %s %s\n' % (openmetrics_name(metric), help_dict[metric])
-            body += '# TYPE %s %s\n' % (openmetrics_name(metric), openmetrics_type(desc))
+            if self.header_flag is True:
+                if self.no_comment is False:
+                    body += '# PCP5 %s %s %s %s %s %s\n' % (openmetrics_name(metric), pmIDStr, get_type_string(desc), pmIndomStr, semantics, units)
+                body += '# TYPE %s %s\n' % (openmetrics_name(metric), openmetrics_type(desc))
+                body += '# HELP %s %s\n' % (openmetrics_name(metric), help_dict[metric])
+                self.header_flag = False
 
             for inst, name, value in results[metric]:
                 if isinstance(value, float):
@@ -515,10 +490,16 @@ def openmetrics_labels(inst, name, desc, labels):
                     value = format(value, fmt)
                 else:
                     str(value)
+
+                if openmetrics_type(desc) == "counter":
+                    openmetrics_name_end = openmetrics_name(metric) + "_total"
+                else:
+                    openmetrics_name_end = openmetrics_name(metric)
+
                 if self.context.type == PM_CONTEXT_ARCHIVE:
-                    body += '%s%s %s %s\n' % (openmetrics_name(metric), openmetrics_labels(inst, name, desc, labels), value, ts)
+                    body += '%s%s %s %s\n' % (openmetrics_name_end, openmetrics_labels(inst, name, desc, labels), value, ts)
                 else:
-                    body += '%s%s %s\n' % (openmetrics_name(metric), openmetrics_labels(inst, name, desc, labels), value)
+                    body += '%s%s %s\n' % (openmetrics_name_end, openmetrics_labels(inst, name, desc, labels), value)
 
         if self.url:
             auth = None
@@ -537,13 +518,13 @@ def openmetrics_labels(inst, name, desc, labels):
         elif self.outfile:
             self.writer.write(body)
         else:
-            print(body)
+            sys.stdout.write(body)
 
     def finalize(self):
         """ Finalize and clean up """
         if self.writer:
             try:
-                self.writer.write("\n")
+                self.writer.write("# EOF\n")
                 self.writer.flush()
             except IOError as write_error:
                 if write_error.errno != errno.EPIPE: