diff --git a/qa/1131 b/qa/1131 index 413da6fe3f..8b8223ca90 100755 --- a/qa/1131 +++ b/qa/1131 @@ -1,6 +1,6 @@ #!/bin/sh # PCP QA Test No. 1131 -# Exercise pcp2json and pcp2openmetrics. +# Exercise pcp2json, pcp2openmetrics, and pcp2opentelemetry. # # Copyright (c) 2017 Red Hat. # @@ -52,6 +52,21 @@ _filter_pcp2openmetrics() | LC_COLLATE=POSIX sort } +_filter_pcp2opentelemetry() +{ + tee -a $here/$seq.full \ + | col -b \ + | sed \ + -e '/\"asDouble\":/ s/[0-9][0-9]*/NCPU/' \ + -e 's/\"'$machineid'\"/MACHINEID/' \ + -e 's/\"'$hostname'\"/HOSTNAME/' \ + -e 's/\"'$domainid'\"/DOMAINID/' \ + -e 's/\"agent\": .*/\"agent\": \"'$OSTYPE'\"/' \ + -e 's/\"userid\": .*/\"userid\": USERID/' \ + -e 's/\"userid\": .*/\"userid\": GROUPID/' \ + -e '/ using stream socket$/d' +} + # real QA test starts here echo "---" pcp2json -a $A -H -I -z "" | _archive_filter @@ -64,6 +79,8 @@ pcp2json -a $A -H -I -z -X -b GB -P 2 -F $tmp.outfile "" echo "---" pcp2openmetrics -s1 -H -z hinv.ncpu | _filter_pcp2openmetrics echo "---" +pcp2opentelemetry -s1 -H -z hinv.ncpu | _filter_pcp2opentelemetry +echo "---" cat $tmp.outfile | _archive_filter diff --git a/qa/1131.out b/qa/1131.out index 098d698a60..47764bf876 100644 --- a/qa/1131.out +++ b/qa/1131.out @@ -2269,10 +2269,65 @@ QA output created by 1131 --- -# HELP hinv_ncpu number of CPUs in the system +# HELP hinv_ncpu b'number of CPUs in the system' # PCP5 hinv_ncpu 60.0.32 u32 PM_INDOM_NULL discrete # TYPE hinv_ncpu gauge +2 hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} NCPU +--- +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "os.type": "linux", + "service.name": "pcp", + "telemetry.sdk.language": "python", + "telemetry.sdk.name": "opentelemetry", + "telemetry.sdk.version": "1.24.0" + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "description": "number of CPUs in the system", + "gauge": { + "dataPoints": [ + { + "asDouble": NCPU, + "attributes": [ + { + "agent": "linux-gnu" + "domainname": DOMAINID, + "groupid": 4209557, + "hostname": HOSTNAME, + "machineid": MACHINEID, + "semantics": "discrete", + "type": "u32", + "userid": GROUPID + } + ], + "startTimeUnixNano": 1714523105.211067 + } + ] + }, + "name": "hinv.ncpu", + "unit": "none" + } + ], + "scope": { + "name": "pcp.hinv", + "version": 1 + } + } + ] + } + ] +} + --- { "@pcp": { diff --git a/qa/1977 b/qa/1977 new file mode 100755 index 0000000000..6fb3b211ab --- /dev/null +++ b/qa/1977 @@ -0,0 +1,81 @@ +#!/bin/sh +# PCP QA Test No. 1827 +# Exercise pcp2opentelemetry HTTP POST functionality. +# +# Copyright (c) 2024 Red Hat. All Rights Reserved. +# + + +seq=`basename $0` +echo "QA output created by $seq" + +. ./common.python + +which pcp2opentelemetry >/dev/null 2>&1 || _notrun "pcp2opentelemetry not installed" + + +_cleanup() +{ + cd $here + $sudo rm -rf $tmp $tmp.* +} + +status=0 # success is the default! +cpus=`pmprobe -v hinv.ncpu | awk '{print $3}'` +hostname=`hostname` +machineid=`_machine_id` +domainid=`_domain_name` +$sudo rm -rf $tmp $tmp.* $seq.full +trap "_cleanup; exit \$status" 0 1 2 3 15 + + +_filter_pcp2opentelemetry_http() +{ + tee -a $here/$seq.full \ + | col -b \ + | sed \ + -e '/\"asDouble\":/ s/[0-9][0-9]*/NCPU/' \ + -e 's/\"'$machineid'\"/MACHINEID/' \ + -e 's/\"'$hostname'\"/HOSTNAME/' \ + -e 's/\"'$domainid'\"/DOMAINID/' \ + -e 's/\"agent\": .*/\"agent\": \"'$OSTYPE'\"/' \ + -e 's/\"userid\": .*/\"userid\": USERID/' \ + -e 's/\"userid\": .*/\"userid\": GROUPID/' \ + -e 's/- - \[[^]]*\]/- -[DATE]/g' \ + -e 's/.*- -/[IP ADDRESS]/g' \ + -e "s/^\(Host: localhost\):$port/\1:PORT/g" \ + -e 's/^\(Content-Length:\) [1-9][0-9]*/\1 SIZE/g' \ + -e 's/^\(User-Agent: python-requests\).*/\1 VERSION/g' \ + -e 's/^\(Date:\).*/\1 DATE/g' \ + -e 's/\(\"context\":\) [0-9][0-9]*/\1 CTXID/g' \ + -e '/^Accept-Encoding: /d' \ + -e 's/\(\hostname=\): \""$hostname"\"/\1:HOST/g' \ + -e '/^Connection: keep-alive/d' \ + -e '/ using stream socket$/d' +} + + +# real QA test starts here +port=`_find_free_port` +$PCP_PYTHON_PROG $here/src/pythonserver.py $port >$tmp.python.out 2>&1 & +pid=$! +sleep 2 # let server start up + + +echo "pcp2opentelemetry invocation" | tee -a $here/$seq.full +pcp2opentelemetry -s1 -u http://localhost:$port/receive hinv.ncpu >$tmp.json.out 2>$tmp.openjson.err + + +echo "pcp2opentelemetry HTTP POST (sorted):" +_filter_pcp2opentelemetry_http <$tmp.python.out + + +cp $tmp.python.out ~/chilton.txt + + +($signal $pid ) >>$seq.full 2>&1 & + + +# success, all done +exit + diff --git a/qa/1977.out b/qa/1977.out new file mode 100644 index 0000000000..8ad5aa823f --- /dev/null +++ b/qa/1977.out @@ -0,0 +1,74 @@ +QA output created by 1977 +pcp2opentelemetry invocation +pcp2opentelemetry HTTP POST (sorted): +INFO:root:Starting httpd... + +INFO:root:POST request, +Path: /receive +Headers: +Host: localhost:PORT +User-Agent: python-requests VERSION +Accept: */* +Content-Type: application/json +Content-Length: SIZE + + + +Body: +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "os.type": "linux", + "service.name": "pcp", + "telemetry.sdk.language": "python", + "telemetry.sdk.name": "opentelemetry", + "telemetry.sdk.version": "1.24.0" + }, + { + "server.address": "http://localhost:54337/receive" + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "description": "number of CPUs in the system", + "gauge": { + "dataPoints": [ + { + "asDouble": NCPU, + "attributes": [ + { + "agent": "linux-gnu" + "domainname": DOMAINID, + "groupid": 4209557, + "hostname": HOSTNAME, + "machineid": MACHINEID, + "semantics": "discrete", + "type": "u32", + "userid": GROUPID + } + ], + "startTimeUnixNano": 1714519586.062307 + } + ] + }, + "name": "hinv.ncpu", + "unit": "none" + } + ], + "scope": { + "name": "pcp.hinv", + "version": 1 + } + } + ] + } + ] +} + +[IP ADDRESS][DATE] "POST /receive HTTP/1.1" 200 - diff --git a/src/pcp2opentelemetry/GNUmakefile b/src/pcp2opentelemetry/GNUmakefile new file mode 100644 index 0000000000..fc23213e53 --- /dev/null +++ b/src/pcp2opentelemetry/GNUmakefile @@ -0,0 +1,46 @@ +# +# Copyright (c) 2024 Red Hat. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# + +TOPDIR = ../.. +include $(TOPDIR)/src/include/builddefs + +TARGET = pcp2opentelemetry +MAN_SECTION = 1 +MAN_PAGES = $(TARGET).$(MAN_SECTION) +MAN_DEST = $(PCP_MAN_DIR)/man$(MAN_SECTION) +BASHDIR = $(PCP_BASHSHARE_DIR)/completions + +default: $(TARGET).py $(MAN_PAGES) + +default: + +include $(BUILDRULES) + +install: default +ifeq "$(HAVE_PYTHON)" "true" + $(INSTALL) -m 755 $(TARGET).py $(PCP_BIN_DIR)/$(TARGET) + $(INSTALL) -S $(BASHDIR)/pcp $(BASHDIR)/$(TARGET) + @$(INSTALL_MAN) +endif + +default_pcp: default + +install_pcp: install + +check:: $(TARGET).py + $(PYLINT) $^ + +check :: $(MAN_PAGES) + $(MANLINT) $^ + diff --git a/src/pcp2opentelemetry/pcp2opentelemetry.1 b/src/pcp2opentelemetry/pcp2opentelemetry.1 new file mode 100644 index 0000000000..6b7aa1b69d --- /dev/null +++ b/src/pcp2opentelemetry/pcp2opentelemetry.1 @@ -0,0 +1,695 @@ +'\"macro stdmacro +.\" +.\" Copyright (C) 2024 Lauren Chilton +.\" Copyright (C) 2024 Red Hat. +.\" +.\" This program is free software; you can redistribute it and/or modify it +.\" under the terms of the GNU General Public License as published by the +.\" Free Software Foundation; either version 2 of the License, or (at your +.\" option) any later version. +.\" +.\" This program is distributed in the hope that it will be useful, but +.\" WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +.\" or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +.\" for more details. +.\" +.\" +.TH PCP2OPENTELEMETRY 1 "PCP" "Performance Co-Pilot" +.SH NAME +\f3pcp2opentelemetry\f1 \- pcp-to-opentelemetry exporter +.SH SYNOPSIS +\fBpcp2opentelemetry\fP +[\fB\-5CEGHIjLmnrRvVxXz?\fP] +[\fB\-4\fP \fIaction\fP] +[\fB\-8\fP|\fB\-9\fP \fIlimit\fP] +[\fB\-a\fP \fIarchive\fP] +[\fB\-A\fP \fIalign\fP] +[\fB\-\-archive\-folio\fP \fIfolio\fP] +[\fB\-b\fP|\fB\-B\fP \fIspace-scale\fP] +[\fB\-c\fP \fIconfig\fP] +[\fB\-\-container\fP \fIcontainer\fP] +[\fB\-\-daemonize\fP] +[\fB\-e\fP \fIderived\fP] +[\fB\-f\fP \fIformat\fP] +[\fB\-F\fP \fIoutfile\fP] +[\fB\-h\fP \fIhost\fP] +[\fB\-i\fP \fIinstances\fP] +[\fB\-J\fP \fIrank\fP] +[\fB\-K\fP \fIspec\fP] +[\fB\-N\fP \fIpredicate\fP] +[\fB\-o\fP \fItimeout\fP] +[\fB\-O\fP \fIorigin\fP] +[\fB\-p\fP \fIpassword\fP] +[\fB\-P\fP|\fB\-0\fP \fIprecision\fP] +[\fB\-q\fP|\fB\-Q\fP \fIcount-scale\fP] +[\fB\-s\fP \fIsamples\fP] +[\fB\-S\fP \fIstarttime\fP] +[\fB\-t\fP \fIinterval\fP] +[\fB\-T\fP \fIendtime\fP] +[\fB\-u\fP \fIurl\fP] +[\fB\-U\fP \fIusername\fP] +[\fB\-y\fP|\fB\-Y\fP \fItime-scale\fP] +[\fB\-Z\fP \fItimezone\fP] +\fImetricspec\fP +[...] +.SH DESCRIPTION +.B pcp2opentelemetry +is a customizable performance metrics exporter tool from PCP to +opentelemetry - +.I https://opentelemetry.io +- format. +Any available performance metric, live or archived, system and/or +application, can be selected for exporting using either command line +arguments or a configuration file. +.PP +.B pcp2opentelemetry +is a close relative of +.BR pmrep (1). +Refer to +.BR pmrep (1) +for the +.I metricspec +description accepted on +.B pcp2opentelemetry +command line. +See +.BR pmrep.conf (5) +for description of the +.B pcp2opentelemetry.conf +configuration file syntax. +This page describes +.B pcp2opentelemetry +specific options and configuration file differences with +.BR pmrep.conf (5). +.BR pmrep (1) +also lists some usage examples of which most are applicable with +.B pcp2opentelemetry +as well. +.PP +Only the command line options listed on this page are supported, +other options available for +.BR pmrep (1) +are not supported. +.PP +Options via environment values (see +.BR pmGetOptions (3)) +override the corresponding built-in default values (if any). +Configuration file options override the corresponding +environment variables (if any). +Command line options override the corresponding configuration +file options (if any). +.SH CONFIGURATION FILE +.B pcp2opentelemetry +uses a configuration file with syntax described in +.BR pmrep.conf (5). +The following options are common with +.BR pmrep.conf : +.BR version , +.BR source , +.BR speclocal , +.BR derived , +.BR header , +.BR globals , +.BR samples , +.BR interval , +.BR type , +.BR type_prefer , +.BR ignore_incompat , +.BR names_change , +.BR instances , +.BR live_filter , +.BR rank , +.BR limit_filter , +.BR limit_filter_force , +.BR invert_filter , +.BR predicate , +.BR omit_flat , +.BR include_labels , +.BR precision , +.BR precision_force , +.BR count_scale , +.BR count_scale_force , +.BR space_scale , +.BR space_scale_force , +.BR time_scale , +.BR time_scale_force . +The rest of the +.B pmrep.conf +options are recognized but ignored for compatibility. +.SS pcp2opentelemetry specific options +url (string) +.RS 4 +Send OPENTELEMETRY output as a HTTP POST to the given \fBurl\fP. +Corresponding command line option is \fB\-u\fP. +Defaults to \fBNone\fP. +.RE +.PP +http_pass (string) +.RS 4 +Use given password for Basic Authentication when sending a HTTP POST. +Corresponding command line option is \fB\-p\fP. +Defaults to \fBNone\fP. +.RE +.PP +http_user (string) +.RS 4 +Use given username for Basic Authentication when sending a HTTP POST. +Corresponding command line option is \fB\-U\fP. +Defaults to \fBNone\fP. +.RE +.PP +http_timeout (number) +.RS 4 +Maximum time (in seconds) when sending a HTTP POST. +Corresponding command line option is \fB\-o\fP. +Defaults to \fB2.5\fP seconds. +.RE +.SH OPTIONS +The available command line options are: +.TP 5 +\fB\-0\fR \fIprecision\fR, \fB\-\-precision\-force\fR=\fIprecision\fR +Like +.B \-P +but this option \fIwill\fP override per-metric specifications. +.TP +\fB\-4\fR \fIaction\fR, \fB\-\-names\-change\fR=\fIaction\fR +Specify which +.I action +to take on receiving a metric names change event during sampling. +These events occur when a PMDA discovers new metrics sometime +after starting up, and informs running client tools like +.BR pcp2opentelemetry. +Valid values for +.I action +are \fBupdate\fP (refresh metrics being sampled), +\fBignore\fP (do nothing \- the default behaviour) +and \fBabort\fP (exit the program if such an event occurs). +.TP +\fB\-5\fR, \fB\-\-ignore\-unknown\fR +Silently ignore any metric name that cannot be resolved. +At least one metric must be found for the tool to start. +.TP +\fB\-8\fR \fIlimit\fR, \fB\-\-limit\-filter\fR=\fIlimit\fR +Limit results to instances with values above/below +.IR limit . +A positive integer will include instances with values +at or above the limit in reporting. +A negative integer will include instances with values +at or below the limit in reporting. +A value of zero performs no limit filtering. +This option will \fInot\fP override possible per-metric specifications. +See also +.BR \-J " and " +.BR \-N . +.TP +\fB\-9\fR \fIlimit\fR, \fB\-\-limit\-filter\-force\fR=\fIlimit\fR +Like +.B \-8 +but this option \fIwill\fP override per-metric specifications. +.TP +\fB\-a\fR \fIarchive\fR, \fB\-\-archive\fR=\fIarchive\fR +Performance metric values are retrieved from the set of Performance +Co-Pilot (PCP) archive files identified by the +.I archive +argument, which is a comma-separated list of names, each +of which may be the base name of an archive or the name of +a directory containing one or more archives. +.TP +\fB\-A\fR \fIalign\fR, \fB\-\-align\fR=\fIalign\fR +Force the initial sample to be +aligned on the boundary of a natural time unit +.IR align . +Refer to +.BR PCPIntro (1) +for a complete description of the syntax for +.IR align . +.TP +\fB\-\-archive\-folio\fR=\fIfolio\fR +Read metric source archives from the PCP archive +.I folio +created by tools like +.BR pmchart (1) +or, less often, manually with +.BR mkaf (1). +.TP +\fB\-b\fR \fIscale\fR, \fB\-\-space\-scale\fR=\fIscale\fR +.I Unit/scale +for space (byte) metrics, possible values include +.BR bytes , +.BR Kbytes , +.BR KB , +.BR Mbytes , +.BR MB , +and so forth. +This option will \fInot\fP override possible per-metric specifications. +See also +.BR pmParseUnitsStr (3). +.TP +\fB\-B\fR \fIscale\fR, \fB\-\-space\-scale\-force\fR=\fIscale\fR +Like +.B \-b +but this option \fIwill\fP override per-metric specifications. +.TP +\fB\-c\fR \fIconfig\fR, \fB\-\-config\fR=\fIconfig\fR +Specify the +.I config +file or directory to use. +In case \fIconfig\fP is a directory all files in it ending +\fB.conf\fR will be included. +The default is the first found of: +.IR ./pcp2opentelemetry.conf , +.IR \f(CR$HOME\fP/.pcp2opentelemetry.conf , +.IR \f(CR$HOME\fP/pcp/pcp2opentelemetry.conf , +and +.IR \f(CR$PCP_SYSCONF_DIR\fP/pcp2opentelemetry.conf . +For details, see the above section and +.BR pmrep.conf (5). +.TP +\fB\-\-container\fR=\fIcontainer\fR +Fetch performance metrics from the specified +.IR container , +either local or remote (see +.BR \-h ). +.TP +\fB\-C\fR, \fB\-\-check\fR +Exit before reporting any values, but after parsing the configuration +and metrics and printing possible headers. +.TP +.B \-\-daemonize +Daemonize on startup. +.TP +\fB\-e\fR \fIderived\fR, \fB\-\-derived\fR=\fIderived\fR +Specify +.I derived +performance metrics. +If +.I derived +starts with a slash (``/'') or with a dot (``.'') it will be +interpreted as a PCP derived metrics configuration file, otherwise it will +be interpreted as comma- or semicolon-separated derived metric expressions. +For complete description of derived metrics and PCP derived metrics +configuration files see +.BR pmLoadDerivedConfig (3) +and +.BR pmRegisterDerived (3). +Alternatively, using +.BR pmrep.conf (5) +configuration syntax allows defining derived metrics as part of metricsets. +.TP +\fB\-E\fR, \fB\-\-exact\-types\fR +Write numbers as number data types, not as strings, potentially +losing some precision. +.TP +\fB\-f\fR \fIformat\fR, \fB\-\-timestamp\-format\fR=\fIformat\fR +Use the +.I format +string for formatting the timestamp. +The format will be used with Python's +.B datetime.strftime +method which is mostly the same as that described in +.BR strftime (3). +The default is +.BR "%Y-%m-%d %H:%M:%S" . +.TP +\fB\-F\fR \fIoutfile\fR, \fB\-\-output\-file\fR=\fIoutfile\fR +Specify the output file +.IR outfile . +.TP +\fB\-G\fR, \fB\-\-no\-globals\fR +Do not include global metrics in reporting (see +.BR pmrep.conf (5)). +.TP +\fB\-h\fR \fIhost\fR, \fB\-\-host\fR=\fIhost\fR +Fetch performance metrics from +.BR pmcd (1) +on +.IR host , +rather than from the default localhost. +.TP +\fB\-H\fR, \fB\-\-no\-header\fR +Do not print any headers. +.TP +\fB\-i\fR \fIinstances\fR, \fB\-\-instances\fR=\fIinstances\fR +Retrieve and report only the specified metric +.IR instances . +By default all instances, present and future, are reported. +.RS +.PP +Refer to +.BR pmrep (1) +for complete description of this option. +.RE +.TP +\fB\-I\fR, \fB\-\-ignore\-incompat\fR +Ignore incompatible metrics. +By default incompatible metrics (that is, +their type is unsupported or they cannot be scaled as requested) +will cause +.B pcp2opentelemetry +to terminate with an error message. +With this option all incompatible metrics are silently omitted +from reporting. +This may be especially useful when requesting +non-leaf nodes of the PMNS tree for reporting. +.TP +\fB\-j\fR, \fB\-\-live\-filter\fR +Perform instance live filtering. +This allows capturing all named instances even if processes +are restarted at some point (unlike without live filtering). +Performing live filtering over a huge number of instances will add +some internal overhead so a bit of user caution is advised. +See also +.BR \-n . +.TP +\fB\-J\fR \fIrank\fR, \fB\-\-rank\fR=\fIrank\fR +Limit results to highest/lowest +.IR rank ed +instances of set-valued metrics. +A positive integer will include highest valued instances in reporting. +A negative integer will include lowest valued instances in reporting. +A value of zero performs no ranking. +Ranking does not imply sorting, see +.BR \-6 . +See also +.BR \-8 . +.TP +\fB\-K\fR \fIspec\fR, \fB\-\-spec\-local\fR=\fIspec\fR +When fetching metrics from a local context (see +.BR \-L ), +the +.B \-K +option may be used to control the DSO PMDAs that should be made accessible. +The +.I spec +argument conforms to the syntax described in +.BR pmSpecLocalPMDA (3). +More than one +.B \-K +option may be used. +.TP +\fB\-L\fR, \fB\-\-local\-PMDA\fR +Use a local context to collect metrics from DSO PMDAs on the local host +without PMCD. +See also +.BR \-K . +.TP +\fB\-n\fR, \fB\-\-invert\-filter\fR +Perform ranking before live filtering. +By default instance live filtering (when requested, see +.BR \-j ) +happens before instance ranking (when requested, see +.BR \-J ). +With this option the logic is inverted and ranking happens before +live filtering. +.TP +\fB\-m\fR, \fB\-\-include\-labels\fR +Include PCP metric labels in the output. +.TP +\fB\-N\fR \fIpredicate\fR, \fB\-\-predicate\fR=\fIpredicate\fR +Specify a comma-separated list of +.I predicate +filter reference metrics. +By default ranking (see +.BR \-J ) +happens for each metric individually. +With predicates, ranking is done only for the +specified predicate metrics. +When reporting, rest of the metrics sharing the same +.I instance domain +(see +.BR PCPIntro (1)) +as the predicate will include only the highest/lowest ranking +instances of the corresponding predicate. +Ranking does not imply sorting, see +.BR \-6 . +.RS +.PP +So for example, using \fBproc.memory.rss\fP +(resident memory size of process) +as the +.I predicate +metric together with \fBproc.io.total_bytes\fP and \fBmem.util.used\fP as +metrics to be reported, only the processes using most/least (as per +.BR \-J ) +memory will be included when reporting total bytes written by processes. +Since \fBmem.util.used\fP is a single-valued metric (thus not sharing the +same instance domain as the process related metrics), +it will be reported as usual. +.RE +.TP +\fB\-o\fR, \fB\-\-http-timeout\fR +Timeout (in seconds) when sending a HTTP POST with the +.BR \-u +option. +Default value is \fB2.5\fP seconds. +.TP +\fB\-O\fR \fIorigin\fR, \fB\-\-origin\fR=\fIorigin\fR +When reporting archived metrics, start reporting at +.I origin +within the time window (see +.B \-S +and +.BR \-T ). +Refer to +.BR PCPIntro (1) +for a complete description of the syntax for +.IR origin . +.TP +\fB\-p\fR, \fB\-\-http-pass\fR +Password when using HTTP basic authentication with the +.BR \-u +option. +.TP +\fB\-P\fR \fIprecision\fR, \fB\-\-precision\fR=\fIprecision\fR +Use +.I precision +for numeric non-integer output values. +The default is to use 3 decimal places (when applicable). +This option will \fInot\fP override possible per-metric specifications. +.TP +\fB\-q\fR \fIscale\fR, \fB\-\-count\-scale\fR=\fIscale\fR +.I Unit/scale +for count metrics, possible values include +.BR "count x 10^\-1" , +.BR "count" , +.BR "count x 10" , +.BR "count x 10^2" , +and so forth from +.B 10^\-8 +to +.BR 10^7 . +.\" https://bugzilla.redhat.com/show_bug.cgi?id=1264124 +(These values are currently space-sensitive.) +This option will \fInot\fP override possible per-metric specifications. +See also +.BR pmParseUnitsStr (3). +.TP +\fB\-Q\fR \fIscale\fR, \fB\-\-count\-scale\-force\fR=\fIscale\fR +Like +.B \-q +but this option \fIwill\fP override per-metric specifications. +.TP +\fB\-r\fR, \fB\-\-raw\fR +Output raw metric values, do not convert cumulative counters to rates. +This option \fIwill\fP override possible per-metric specifications. +.TP +\fB\-R\fR, \fB\-\-raw\-prefer\fR +Like +.B \-r +but this option will \fInot\fP override per-metric specifications. +.TP +\fB\-s\fR \fIsamples\fR, \fB\-\-samples\fR=\fIsamples\fR +The +.I samples +argument defines the number of samples to be retrieved and reported. +If +.I samples +is 0 or +.B \-s +is not specified, +.B pcp2opentelemetry +will sample and report continuously (in real time mode) or until the end +of the set of PCP archives (in archive mode). +See also +.BR \-T . +.TP +\fB\-S\fR \fIstarttime\fR, \fB\-\-start\fR=\fIstarttime\fR +When reporting archived metrics, the report will be restricted to those +records logged at or after +.IR starttime . +Refer to +.BR PCPIntro (1) +for a complete description of the syntax for +.IR starttime . +.TP +\fB\-t\fR \fIinterval\fR, \fB\-\-interval\fR=\fIinterval\fR +Set the reporting +.I interval +to something other than the default 1 second. +The +.I interval +argument follows the syntax described in +.BR PCPIntro (1), +and in the simplest form may be an unsigned integer +(the implied units in this case are seconds). +See also the +.B \-T +option. +.TP +\fB\-T\fR \fIendtime\fR, \fB\-\-finish\fR=\fIendtime\fR +When reporting archived metrics, the report will be restricted to those +records logged before or at +.IR endtime . +Refer to +.BR PCPIntro (1) +for a complete description of the syntax for +.IR endtime . +.RS +.PP +When used to define the runtime before \fBpcp2opentelemetry\fP will exit, +if no \fIsamples\fP is given (see \fB\-s\fP) then the number of +reported samples depends on \fIinterval\fP (see \fB\-t\fP). +If +.I samples +is given then +.I interval +will be adjusted to allow reporting of +.I samples +during runtime. +In case all of +.BR \-T , +.BR \-s , +and +.B \-t +are given, +.I endtime +determines the actual time +.B pcp2opentelemetry +will run. +.RE +.TP +\fB\-u\fR, \fB\-\-url\fR +URL for sending an HTTP POST (instead of default standard output). +.TP +\fB\-U\fR, \fB\-\-http-user\fR +Username when using HTTP basic authentication with the +.BR \-u +option. +.TP +\fB\-v\fR, \fB\-\-omit\-flat\fR +Report only set-valued metrics with instances (e.g. disk.dev.read) and +omit single-valued ``flat'' metrics without instances (e.g. +kernel.all.sysfork). +See +.B \-i +and +.BR \-I . +.TP +\fB\-V\fR, \fB\-\-version\fR +Display version number and exit. +.TP +\fB\-x\fR, \fB\-\-with\-extended\fR +Write extended information. +.TP +\fB\-X\fR, \fB\-\-with\-everything\fR +Write everything known about metrics, including PCP internal IDs. +Labels are, however, omitted for backward compatibility, +use \fB\-m\fP to include them as well. +.TP +\fB\-y\fR \fIscale\fR, \fB\-\-time\-scale\fR=\fIscale\fR +.I Unit/scale +for time metrics, possible values include +.BR nanosec , +.BR ns , +.BR microsec , +.BR us , +.BR millisec , +.BR ms , +and so forth up to +.BR hour , +.BR hr . +This option will \fInot\fP override possible per-metric specifications. +See also +.BR pmParseUnitsStr (3). +.TP +\fB\-Y\fR \fIscale\fR, \fB\-\-time\-scale\-force\fR=\fIscale\fR +Like +.B \-y +but this option \fIwill\fP override per-metric specifications. +.TP +\fB\-z\fR, \fB\-\-hostzone\fR +Use the local timezone of the host that is the source of the +performance metrics, as identified by either the +.B \-h +or the +.B \-a +options. +The default is to use the timezone of the local host. +.TP +\fB\-Z\fR \fItimezone\fR, \fB\-\-timezone\fR=\fItimezone\fR +Use +.I timezone +for the date and time. +.I Timezone +is in the format of the environment variable +.B TZ +as described in +.BR environ (7). +Note that when including a timezone string in output, ISO 8601 -style +UTC offsets are used (so something like \-Z EST+5 will become UTC-5). +.TP +\fB\-?\fR, \fB\-\-help\fR +Display usage message and exit. +.SH FILES +.TP 5 +.I pcp2opentelemetry.conf +\fBpcp2opentelemetry\fP configuration file (see \fB\-c\fP) +.TP +.I \f(CR$PCP_SYSCONF_DIR\fP/pmrep/*.conf +system provided default \fBpmrep\fP configuration files +.SH PCP ENVIRONMENT +Environment variables with the prefix \fBPCP_\fP are used to parameterize +the file and directory names used by PCP. +On each installation, the +file \fI/etc/pcp.conf\fP contains the local values for these variables. +The \fB$PCP_CONF\fP variable may be used to specify an alternative +configuration file, as described in \fBpcp.conf\fP(5). +.PP +For environment variables affecting PCP tools, see \fBpmGetOptions\fP(3). +.SH SEE ALSO +.BR PCPIntro (1), +.BR mkaf (1), +.BR pcp (1), +.BR pcp2elasticsearch (1), +.BR pcp2graphite (1), +.BR pcp2influxdb (1), +.BR pcp2openmetrics (1), +.BR pcp2spark (1), +.BR pcp2xlsx (1), +.BR pcp2xml (1), +.BR pcp2json (1), +.BR pcp2zabbix (1), +.BR pmcd (1), +.BR pminfo (1), +.BR pmrep (1), +.BR pmGetOptions (3), +.BR pmLoadDerivedConfig (3), +.BR pmParseUnitsStr (3), +.BR pmRegisterDerived (3), +.BR pmSpecLocalPMDA (3), +.BR LOGARCHIVE (5), +.BR pcp.conf (5), +.BR pmrep.conf (5), +.BR PMNS (5) +and +.BR environ (7). + +.\" control lines for scripts/man-spell +.\" +ok+ limit_filter_force count_scale_force space_scale_force +.\" +ok+ CEGHIjLmnrRvVxXz {from -5CEGHIjLmnrRvVxXz confuses ispell} +.\" +ok+ time_scale_force ignore_incompat precision_force +.\" +ok+ include_labels invert_filter names_change limit_filter +.\" +ok+ http_timeout live_filter total_bytes count_scale +.\" +ok+ space_scale exact_types type_prefer metricsets time_scale +.\" +ok+ omit_flat http_pass http_user datetime incompat influxdb +.\" +ok+ IDs EST diff --git a/src/pcp2opentelemetry/pcp2opentelemetry.py b/src/pcp2opentelemetry/pcp2opentelemetry.py new file mode 100644 index 0000000000..ba45ec278e --- /dev/null +++ b/src/pcp2opentelemetry/pcp2opentelemetry.py @@ -0,0 +1,660 @@ +#!/usr/bin/env pmpython +# +# Copyright (C) 2024 Lauren Chilton +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. + + +""" PCP to OPENTELEMETRY Bridge """ + +# Common imports +from collections import OrderedDict +import errno +import time +import sys +import platform + +# Our imports +import requests +import os +import cpmapi +import json + +# PCP Python PMAPI +from pcp import pmapi, pmconfig +from cpmapi import PM_CONTEXT_ARCHIVE, PM_INDOM_NULL, PM_DEBUG_APPL1, PM_TIME_SEC +# Default config +DEFAULT_CONFIG = ["./pcp2opentelemetry.conf", "$HOME/.pcp2opentelemetry.conf", + "$HOME/.pcp/pcp2opentelemetry.conf", "$PCP_SYSCONF_DIR/pcp2opentelemetry.conf"] + +# Defaults +CONFVER = 1 +INDENT = 2 +TIMEFMT = "%Y-%m-%d %H:%M:%S" +TIMEOUT = 2.5 # seconds + +class PCP2OPENTELEMETRY(object): + """ PCP to OPENTELEMETRY """ + def __init__(self): + """ Construct object, prepare for command line handling """ + self.context = None + self.daemonize = 0 + self.pmconfig = pmconfig.pmConfig(self) + self.opts = self.options() + + # Configuration directives + self.keys = ('source', 'output', 'derived', 'header', 'globals', + 'samples', 'interval', 'type', 'precision', 'daemonize', + 'timefmt', 'everything', + 'count_scale', 'space_scale', 'time_scale', 'version', + 'count_scale_force', 'space_scale_force', 'time_scale_force', + 'type_prefer', 'precision_force', 'limit_filter', 'limit_filter_force', + 'live_filter', 'rank', 'invert_filter', 'predicate', 'names_change', + 'speclocal', 'instances', 'ignore_incompat', 'ignore_unknown', + 'omit_flat', 'include_labels', 'url', 'http_user', 'http_pass', + 'http_timeout') + + # Ignored for pmrep(1) compatibility + self.keys_ignore = ( + 'timestamp', 'unitinfo', 'colxrow', 'separate_header', 'fixed_header', + 'delay', 'width', 'delimiter', 'extcsv', 'width_force', + 'extheader', 'repeat_header', 'interpol', + 'dynamic_header', 'overall_rank', 'overall_rank_alt', 'sort_metric', + 'instinfo', 'include_texts') + + # The order of preference for options (as present): + # 1 - command line options + # 2 - options from configuration file(s) + # 3 - built-in defaults defined below + self.check = 0 + self.version = CONFVER + self.source = "local:" + self.output = None # For pmrep conf file compat only + self.speclocal = None + self.derived = None + self.header = 1 + self.globals = 1 + self.samples = None # forever + self.interval = pmapi.timeval(10) # 10 sec + self.opts.pmSetOptionInterval(str(10)) # 10 sec + self.delay = 0 + self.type = 0 + self.type_prefer = self.type + self.ignore_incompat = 0 + self.ignore_unknown = 0 + self.names_change = 0 # ignore + self.instances = [] + self.live_filter = 0 + self.rank = 0 + self.limit_filter = 0 + self.limit_filter_force = 0 + self.invert_filter = 0 + self.predicate = None + self.omit_flat = 0 + self.include_labels = 0 + self.precision = 3 # .3f + self.precision_force = None + self.timefmt = TIMEFMT + self.interpol = 0 + self.count_scale = None + self.count_scale_force = None + self.space_scale = None + self.space_scale_force = None + self.time_scale = None + self.time_scale_force = None + + # Not in pcp2openmetrics.conf, won't overwrite + self.outfile = None + + self.everything = 0 + self.url = None + self.http_user = None + self.http_pass = None + self.http_timeout = TIMEOUT + + # Internal + self.runtime = -1 + + self.data = None + self.prev_ts = None + self.writer = None + + # Performance metrics store + # key - metric name + # values - 0:txt label, 1:instance(s), 2:unit/scale, 3:type, + # 4:width, 5:pmfg item, 6:precision, 7:limit + self.metrics = OrderedDict() + self.pmfg = None + self.pmfg_ts = None + + # Read configuration and prepare to connect + self.config = self.pmconfig.set_config_path(DEFAULT_CONFIG) + self.pmconfig.read_options() + self.pmconfig.read_cmd_line() + self.pmconfig.prepare_metrics() + self.pmconfig.set_signal_handler() + + def options(self): + """ Setup default command line argument option handling """ + opts = pmapi.pmOptions() + opts.pmSetOptionCallback(self.option) + opts.pmSetOverrideCallback(self.option_override) + opts.pmSetShortOptions("a:h:LK:c:Ce:D:V?HGA:S:T:O:s:t:rRIi:jJ:4:58:9:nN:vmP:0:q:b:y:Q:B:Y:F:f:Z:zo:p:U:u:") + opts.pmSetShortUsage("[option...] metricspec [...]") + + opts.pmSetLongOptionHeader("General options") + opts.pmSetLongOptionArchive() # -a/--archive + opts.pmSetLongOptionArchiveFolio() # --archive-folio + opts.pmSetLongOptionContainer() # --container + opts.pmSetLongOptionHost() # -h/--host + opts.pmSetLongOptionLocalPMDA() # -L/--local-PMDA + opts.pmSetLongOptionSpecLocal() # -K/--spec-local + opts.pmSetLongOption("config", 1, "c", "FILE", "config file path") + opts.pmSetLongOption("check", 0, "C", "", "check config and metrics and exit") + opts.pmSetLongOption("output-file", 1, "F", "OUTFILE", "output file") + opts.pmSetLongOption("derived", 1, "e", "FILE|DFNT", "derived metrics definitions") + opts.pmSetLongOption("daemonize", 0, "", "", "daemonize on startup") + opts.pmSetLongOptionDebug() # -D/--debug + opts.pmSetLongOptionVersion() # -V/--version + opts.pmSetLongOptionHelp() # -?/--help + + opts.pmSetLongOptionHeader("Reporting options") + opts.pmSetLongOption("no-header", 0, "H", "", "omit headers") + opts.pmSetLongOption("no-globals", 0, "G", "", "omit global metrics") + opts.pmSetLongOptionAlign() # -A/--align + opts.pmSetLongOptionStart() # -S/--start + opts.pmSetLongOptionFinish() # -T/--finish + opts.pmSetLongOptionOrigin() # -O/--origin + opts.pmSetLongOptionSamples() # -s/--samples + opts.pmSetLongOptionInterval() # -t/--interval + opts.pmSetLongOptionTimeZone() # -Z/--timezone + opts.pmSetLongOptionHostZone() # -z/--hostzone + opts.pmSetLongOption("raw", 0, "r", "", "output raw counter values (no rate conversion)") + opts.pmSetLongOption("raw-prefer", 0, "R", "", "prefer output raw counter values (no rate conversion)") + opts.pmSetLongOption("ignore-incompat", 0, "I", "", "ignore incompatible instances (default: abort)") + opts.pmSetLongOption("ignore-unknown", 0, "5", "", "ignore unknown metrics (default: abort)") + opts.pmSetLongOption("names-change", 1, "4", "ACTION", "update/ignore/abort on PMNS change (default: ignore)") + opts.pmSetLongOption("instances", 1, "i", "STR", "instances to report (default: all current)") + opts.pmSetLongOption("live-filter", 0, "j", "", "perform instance live filtering") + opts.pmSetLongOption("rank", 1, "J", "COUNT", "limit results to COUNT highest/lowest valued instances") + opts.pmSetLongOption("limit-filter", 1, "8", "LIMIT", "default limit for value filtering") + opts.pmSetLongOption("limit-filter-force", 1, "9", "LIMIT", "forced limit for value filtering") + opts.pmSetLongOption("invert-filter", 0, "n", "", "perform ranking before live filtering") + opts.pmSetLongOption("predicate", 1, "N", "METRIC", "set predicate filter reference metric") + opts.pmSetLongOption("omit-flat", 0, "v", "", "omit single-valued metrics") + opts.pmSetLongOption("include-labels", 0, "m", "", "include metric label info") + opts.pmSetLongOption("timestamp-format", 1, "f", "STR", "strftime string for timestamp format") + opts.pmSetLongOption("precision", 1, "P", "N", "prefer N digits after decimal separator (default: 3)") + opts.pmSetLongOption("precision-force", 1, "0", "N", "force N digits after decimal separator") + opts.pmSetLongOption("count-scale", 1, "q", "SCALE", "default count unit") + opts.pmSetLongOption("count-scale-force", 1, "Q", "SCALE", "forced count unit") + opts.pmSetLongOption("space-scale", 1, "b", "SCALE", "default space unit") + opts.pmSetLongOption("space-scale-force", 1, "B", "SCALE", "forced space unit") + opts.pmSetLongOption("time-scale", 1, "y", "SCALE", "default time unit") + opts.pmSetLongOption("time-scale-force", 1, "Y", "SCALE", "forced time unit") + + opts.pmSetLongOption("url", 1, "u", "URL", "URL of endpoint to receive HTTP POST") + opts.pmSetLongOption("http-timeout", 1, "o", "SECONDS", "timeout when sending HTTP POST") + opts.pmSetLongOption("http-pass", 1, "p", "PASSWORD", "password for endpoint") + opts.pmSetLongOption("http-user", 1, "U", "USERNAME", "username for endpoint") + + return opts + + def option_override(self, opt): + """ Override standard PCP options """ + if opt in ('g', 'H', 'K', 'n', 'N', 'p'): + return 1 + return 0 + + def option(self, opt, optarg, _index): + """ Perform setup for individual command line option """ + if opt == 'daemonize': + self.daemonize = 1 + elif opt == 'K': + if not self.speclocal or not self.speclocal.startswith(";"): + self.speclocal = ";" + optarg + else: + self.speclocal = self.speclocal + ";" + optarg + elif opt == 'c': + self.config = optarg + elif opt == 'C': + self.check = 1 + elif opt == 'F': + if os.path.exists(optarg): + sys.stderr.write("File %s already exists.\n" % optarg) + sys.exit(1) + self.outfile = optarg + elif opt == 'e': + if not self.derived or not self.derived.startswith(";"): + self.derived = ";" + optarg + else: + self.derived = self.derived + ";" + optarg + elif opt == 'H': + self.header = 0 + elif opt == 'G': + self.globals = 0 + elif opt == 'r': + self.type = 1 + elif opt == 'R': + self.type_prefer = 1 + elif opt == 'I': + self.ignore_incompat = 1 + elif opt == '5': + self.ignore_unknown = 1 + elif opt == '4': + if optarg == 'ignore': + self.names_change = 0 + elif optarg == 'abort': + self.names_change = 1 + elif optarg == 'update': + self.names_change = 2 + else: + sys.stderr.write("Unknown names-change action '%s' specified.\n" % optarg) + sys.exit(1) + elif opt == 'i': + self.instances = self.instances + self.pmconfig.parse_instances(optarg) + elif opt == 'j': + self.live_filter = 1 + elif opt == 'J': + self.rank = optarg + elif opt == '8': + self.limit_filter = optarg + elif opt == '9': + self.limit_filter_force = optarg + elif opt == 'n': + self.invert_filter = 1 + elif opt == 'N': + self.predicate = optarg + elif opt == 'v': + self.omit_flat = 1 + elif opt == 'm': + self.include_labels = 1 + elif opt == 'P': + self.precision = optarg + elif opt == '0': + self.precision_force = optarg + elif opt == 'f': + self.timefmt = optarg + elif opt == 'q': + self.count_scale = optarg + elif opt == 'Q': + self.count_scale_force = optarg + elif opt == 'b': + self.space_scale = optarg + elif opt == 'B': + self.space_scale_force = optarg + elif opt == 'y': + self.time_scale = optarg + elif opt == 'Y': + self.time_scale_force = optarg + elif opt == 'u': + self.url = optarg + elif opt == 'o': + self.http_timeout = float(optarg) + elif opt == 'U': + self.http_user = optarg + elif opt == 'P': + self.http_pass = optarg + else: + raise pmapi.pmUsageErr() + + def connect(self): + """ Establish PMAPI context """ + context, self.source = pmapi.pmContext.set_connect_options(self.opts, self.source, self.speclocal) + + self.pmfg = pmapi.fetchgroup(context, self.source) + self.pmfg_ts = self.pmfg.extend_timestamp() + self.context = self.pmfg.get_context() + + if pmapi.c_api.pmSetContextOptions(self.context.ctx, self.opts.mode, self.opts.delta): + raise pmapi.pmUsageErr() + + def validate_config(self): + """ Validate configuration options """ + if self.version != CONFVER: + sys.stderr.write("Incompatible configuration file version (read v%s, need v%d).\n" % + (self.version, CONFVER)) + sys.exit(1) + + self.pmconfig.validate_common_options() + + self.pmconfig.validate_metrics(curr_insts=not self.live_filter) + self.pmconfig.finalize_options() + + def execute(self): + """ Fetch and report """ + # Debug + if self.context.pmDebug(PM_DEBUG_APPL1): + sys.stdout.write("Known config file keywords: " + str(self.keys) + "\n") + sys.stdout.write("Known metric spec keywords: " + str(self.pmconfig.metricspec) + "\n") + + # Set delay mode, interpolation + if self.context.type != PM_CONTEXT_ARCHIVE: + self.delay = 1 + self.interpol = 1 + + # Common preparations + self.context.prepare_execute(self.opts, False, self.interpol, self.interval) + + # Headers + if self.header == 1: + self.header = 0 + self.write_header() + + # Just checking + if self.check == 1: + return + + # Daemonize when requested + if self.daemonize == 1: + self.opts.daemonize() + + # Align poll interval to host clock + if self.context.type != PM_CONTEXT_ARCHIVE and self.opts.pmGetOptionAlignment(): + align = float(self.opts.pmGetOptionAlignment()) - (time.time() % float(self.opts.pmGetOptionAlignment())) + time.sleep(align) + + # Main loop + refresh_metrics = 0 + while self.samples != 0: + # Refresh metrics as needed + if refresh_metrics: + refresh_metrics = 0 + self.pmconfig.update_metrics(curr_insts=not self.live_filter) + + # Fetch values + refresh_metrics = self.pmconfig.fetch() + if refresh_metrics < 0: + break + + # Report and prepare for the next round + self.report(self.pmfg_ts()) + if self.samples and self.samples > 0: + self.samples -= 1 + if self.delay and self.interpol and self.samples != 0: + self.pmconfig.pause() + + # Allow to flush buffered values / say goodbye + self.report(None) + + def report(self, tstamp): + """ Report metric values """ + if tstamp is not None: + tstamp = tstamp.strftime(self.timefmt) + + self.write_opentelemetry(tstamp) + + def write_header(self): + """ Write info header """ + output = self.outfile if self.outfile else "stdout" + if self.context.type == PM_CONTEXT_ARCHIVE: + sys.stdout.write('{ "//": "Writing %d archived metrics to %s..." }\n{ "//": "(Ctrl-C to stop)" }\n' + % (len(self.metrics), output)) + return + + sys.stdout.write('{ "//": "Waiting for %d metrics to be written to %s' % (len(self.metrics), output)) + if self.runtime != -1: + sys.stdout.write(':" }\n{ "//": "%s samples(s) with %.1f sec interval ~ %d sec runtime." }\n' % + (self.samples, float(self.interval), self.runtime)) + elif self.samples: + duration = (self.samples - 1) * float(self.interval) + sys.stdout.write(':" }\n{ "//": "%s samples(s) with %.1f sec interval ~ %d sec runtime." }\n' % + (self.samples, float(self.interval), duration)) + else: + sys.stdout.write('..." }\n{ "//": "(Ctrl-C to stop)" }\n') + + def write_opentelemetry(self, timestamp): + """ Write results in opentelemetry format """ + if timestamp is None: + # Silent goodbye, close in finalize() + return + + ts = self.context.datetime_to_secs(self.pmfg_ts(), PM_TIME_SEC) + + if self.prev_ts is None: + self.prev_ts = ts + + if not self.writer and not self.url: + if self.outfile is None: + self.writer = sys.stdout + else: + self.writer = open(self.outfile, 'wt') + + results = self.pmconfig.get_ranked_results(valid_only=True) + def get_type_string(desc): + """ Get metric type as string """ + if desc.contents.type == pmapi.c_api.PM_TYPE_32: + mtype = "32" + elif desc.contents.type == pmapi.c_api.PM_TYPE_U32: + mtype = "u32" + elif desc.contents.type == pmapi.c_api.PM_TYPE_64: + mtype = "64" + elif desc.contents.type == pmapi.c_api.PM_TYPE_U64: + mtype = "u64" + elif desc.contents.type == pmapi.c_api.PM_TYPE_FLOAT: + mtype = "float" + elif desc.contents.type == pmapi.c_api.PM_TYPE_DOUBLE: + mtype = "float" + elif desc.contents.type == pmapi.c_api.PM_TYPE_STRING: + mtype = "string" + else: + mtype = "unknown" + return mtype + + def data_attribute_funct(labels, context, desc, inst, name): + attribute_list = [] + new_dict = {} + new_dict["semantics"] = self.context.pmSemStr(desc.contents.sem) + new_dict["type"] = get_type_string(desc) + if desc.indom != PM_INDOM_NULL: + new_dict["instname"] = name + new_dict["instid"] = inst + for key in labels: + new_dict.update(labels[key]) + attribute_list.append(new_dict) + return attribute_list + + def data_points_function(metric, results, labels, context, desc): + numdatapoints_list = [] + for inst, name, value in results[metric]: + datapoint_dict = {} + datapoint_dict["asDouble"] = value + if inst != PM_INDOM_NULL: + datapoint_dict["instance"] = inst + datapoint_dict["startTimeUnixNano"] = ts + datapoint_dict["attributes"] = data_attribute_funct(labels, context, desc, inst, name) + numdatapoints_list.append(datapoint_dict) + return numdatapoints_list + + def attribute_function(): + attribute_list = [] + body = {} + body["os.type"] = platform.system().lower() + body["telemetry.sdk.language"] = "python" + body["telemetry.sdk.name"] = "opentelemetry" + body["telemetry.sdk.version"] = "1.24.0" + body["service.name"] = "pcp" + attribute_list.append(body) + if self.url: + url_dict = {} + url_dict["server.address"] = self.url + attribute_list.append(url_dict) + return attribute_list + + def sum_function(metric, results, labels, context, desc): + sum_body = {} + sum_body["aggregationTemporality"] = 1 + sum_body["isMonotonic"] = 'true' + sum_body["dataPoints"] = data_points_function(metric, results, labels, context, desc) + return sum_body + + def gauge_function(metric, results, labels, context, desc): + gauge_body = {} + gauge_body["dataPoints"] = data_points_function(metric, results, labels, context, desc) + return gauge_body + + def scope_function(metric): + split_metric = metric.split(".") + scope_body = {} + scope_body["name"] = "pcp.%s" % split_metric[0] + scope_body["version"] = self.version + return scope_body + + def unit_function(unit): + def dimTime_units(num): + if num == 0: + return "ns" + elif num == 1: + return "us" + elif num == 2: + return "ms" + elif num == 3: + return "s" + elif num == 4: + return "min" + elif num == 5: + return "h" + else: + return "none" + + def dimSpace_units(num): + if num == 0: + return "By" + elif num == 1: + return "KiBy" + elif num == 2: + return "MiBy" + elif num == 3: + return "GiBy" + elif num == 4: + return "TiBy" + elif num == 5: + return "PiBy" + elif num == 6: + return "EiBy" + elif num == 7: + return "ZiBy" + elif num == 8: + return "YiBy" + else: + return "none" + + def dimCount_units(num): + if num == 0: + return "count" + elif num == 1: + return "count x 10" + else: + return "count x 10%d" % num + + if unit.dimTime < 0: + if unit.dimSpace: + return "%s / %s" % (dimSpace_units(unit.scaleSpace), dimTime_units(unit.scaleTime)) + elif unit.dimCount: + return "%s / %s" % (dimCount_units(unit.scaleSpace), dimTime_units(unit.scaleTime)) + elif unit.dimCount < 0: + if unit.dimTime: + return "%s / %s" % (dimTime_units(unit.scaleSpace), dimCount_units(unit.scaleTime)) + elif unit.dimSpace: + return "%s / %s" % (dimSpace_units(unit.scaleSpace), dimCount_units(unit.scaleTime)) + elif unit.dimSpace: + return dimSpace_units(unit.scaleSpace) + elif unit.dimTime: + return dimTime_units(unit.scaleTime) + elif unit.dimCount: + return dimCount_units(unit.scaleCount) + else: + return "none" + + def scopeMetric_function(results): + body = {} + body["metrics"] = [] + for metric in results: + #variable declaration + context = self.pmfg.get_context() + pmid = context.pmLookupName(metric) + i = list(self.metrics.keys()).index(metric) + labels = context.pmLookupLabels(pmid[0]) + desc = self.pmconfig.descs[i] + unit = desc.contents.units + + body["scope"] = scope_function(metric) + metric_dict = {} + metric_dict["name"] = metric + metric_dict["unit"] = unit_function(unit) + metric_dict["description"] = context.pmLookupText(pmid[0]).decode() + if desc.sem == cpmapi.PM_SEM_COUNTER: + metric_dict["sum"] = sum_function(metric, results, labels, context, desc) + else: metric_dict["gauge"] = gauge_function(metric, results, labels, context, desc) + body["metrics"].append(metric_dict) + return body + + self.data = {"resourceMetrics": [{"resource": {"attributes": attribute_function()}, + "scopeMetrics":[scopeMetric_function(results)]}]} + data = json.dumps(self.data, indent=INDENT, sort_keys=True, ensure_ascii=False, separators=(',', ': ')) + + if self.url: + auth = None + if self.http_user and self.http_pass: + auth = requests.auth.HTTPBasicAuth(self.http_user, self.http_pass) + try: + timeout = self.http_timeout + headers = {'Content-Type': 'application/json'} + res = requests.post(self.url, data=data, auth=auth, headers=headers, timeout=timeout) + if res.status_code > 299: + msg = "Cannot send metrics: HTTP code %s\n" % str(res.status_code) + sys.stderr.write(msg) + except requests.exceptions.ConnectionError as post_error: + msg = "Cannot connect to server at %s: %s\n" % (self.url, str(post_error)) + sys.stderr.write(msg) + elif self.outfile: + self.writer.write(data) + else: + print(data) + + def finalize(self): + """ Finalize and clean up """ + if self.writer: + try: + self.writer.write("\n") + self.writer.flush() + except IOError as write_error: + if write_error.errno != errno.EPIPE: + raise + try: + self.writer.close() + except Exception: + pass + self.writer = None + +if __name__ == '__main__': + try: + P = PCP2OPENTELEMETRY() + P.connect() + P.validate_config() + P.execute() + P.finalize() + except pmapi.pmErr as error: + sys.stderr.write("%s: %s" % (error.progname(), error.message())) + if error.message() == "Connection refused": + sys.stderr.write("; is pmcd running?") + sys.stderr.write("\n") + sys.exit(1) + except pmapi.pmUsageErr as usage: + usage.message() + sys.exit(1) + except IOError as error: + if error.errno != errno.EPIPE: + sys.stderr.write("%s\n" % str(error)) + sys.exit(1) + except KeyboardInterrupt: + sys.stdout.write("\n") + P.finalize()