Skip to content

Commit d94a4cc

Browse files
lmchiltonlmchilton
committed
Addition of pcp2opentelemetry tool
Supports translation of pcp metrics into open telemetry format. Supports opentelemetry protocol http json format. Updated relevent qa tests, as well as created a new one (1977) to test the tools http functionality. Added man page and makefile. Contains changes from review comments, including changes to spec and qa files.
1 parent 3be8a8e commit d94a4cc

File tree

13 files changed

+1700
-43
lines changed

13 files changed

+1700
-43
lines changed

Diff for: ‎build/rpm/pcp.spec.in

+25-3
Original file line numberDiff line numberDiff line change
@@ -332,8 +332,8 @@ Requires: pcp-pmda-nvidia-gpu pcp-pmda-roomtemp pcp-pmda-sendmail pcp-pmda-shpin
332332
Requires: pcp-pmda-lustrecomm pcp-pmda-docker pcp-pmda-smart pcp-pmda-farm
333333
Requires: pcp-pmda-hacluster pcp-pmda-sockets pcp-pmda-podman
334334
%if "@have_python@" == "true"
335-
Requires: pcp-geolocate pcp-export-pcp2openmetrics pcp-export-pcp2json
336-
Requires: pcp-export-pcp2spark pcp-export-pcp2xml pcp-export-pcp2zabbix
335+
Requires: pcp-geolocate pcp-export-pcp2openmetrics pcp-export-pcp2opentelemetry
336+
Requires: pcp-export-pcp2json pcp-export-pcp2spark pcp-export-pcp2xml pcp-export-pcp2zabbix
337337
Requires: pcp-pmda-gluster pcp-pmda-zswap pcp-pmda-unbound pcp-pmda-mic
338338
Requires: pcp-pmda-haproxy pcp-pmda-nfsclient pcp-pmda-lmsensors
339339
Requires: pcp-pmda-netcheck pcp-pmda-rabbitmq pcp-pmda-uwsgi
@@ -704,6 +704,24 @@ Requires: %{__python2}-pcp = @package_version@
704704
Performance Co-Pilot (PCP) front-end tools for exporting metric values
705705
in OpenMetrics (https://openmetrics.io/) format.
706706

707+
#
708+
# pcp-export-pcp2opentelemetry
709+
#
710+
%package export-pcp2opentelemetry
711+
License: GPL-2.0-or-later
712+
Summary: Performance Co-Pilot tools for exporting PCP metrics in OpenTelemetry format
713+
URL: https://pcp.io
714+
Requires: pcp-libs >= @package_version@
715+
%if "@enable_python3@" == "true"
716+
Requires: python3-pcp = @package_version@
717+
%else
718+
Requires: %{__python2}-pcp = @package_version@
719+
%endif
720+
721+
%description export-pcp2opentelemetry
722+
Performance Co-Pilot (PCP) front-end tools for exporting metric values
723+
in OpenTelemetry (https://opentelemetry.io/) format.
724+
707725
#
708726
# pcp-export-pcp2spark
709727
#
@@ -2421,6 +2439,7 @@ basic_manifest | keep 'pcp2xlsx' >pcp-export-pcp2xlsx-files
24212439
basic_manifest | keep 'pcp2graphite' >pcp-export-pcp2graphite-files
24222440
basic_manifest | keep 'pcp2json' >pcp-export-pcp2json-files
24232441
basic_manifest | keep 'pcp2openmetrics' >pcp-export-pcp2openmetrics-files
2442+
basic_manifest | keep 'pcp2opentelemetry' >pcp-export-pcp2opentelemetry-files
24242443
basic_manifest | keep 'pcp2spark' >pcp-export-pcp2spark-files
24252444
basic_manifest | keep 'pcp2xml' >pcp-export-pcp2xml-files
24262445
basic_manifest | keep 'pcp2zabbix' >pcp-export-pcp2zabbix-files
@@ -2539,7 +2558,8 @@ done
25392558

25402559
for export_package in \
25412560
pcp2arrow pcp2elasticsearch pcp2graphite pcp2influxdb pcp2json \
2542-
pcp2openmetrics pcp2spark pcp2xlsx pcp2xml pcp2zabbix zabbix-agent ; \
2561+
pcp2openmetrics pcp2opentelemetry pcp2spark pcp2xlsx pcp2xml \
2562+
pcp2zabbix zabbix-agent; \
25432563
do \
25442564
export_packages="$export_packages pcp-export-$export_package"; \
25452565
done
@@ -3258,6 +3278,8 @@ fi
32583278

32593279
%files export-pcp2openmetrics -f pcp-export-pcp2openmetrics-files.rpm
32603280

3281+
%files export-pcp2opentelemetry -f pcp-export-pcp2opentelemetry-files.rpm
3282+
32613283
%files export-pcp2spark -f pcp-export-pcp2spark-files.rpm
32623284

32633285
%files export-pcp2xml -f pcp-export-pcp2xml-files.rpm

Diff for: ‎build/rpm/redhat.spec

+25-3
Original file line numberDiff line numberDiff line change
@@ -568,8 +568,8 @@ Requires: pcp-pmda-bpf
568568
Requires: pcp-pmda-bpftrace
569569
%endif
570570
%if !%{disable_python2} || !%{disable_python3}
571-
Requires: pcp-geolocate pcp-export-pcp2openmetrics pcp-export-pcp2json
572-
Requires: pcp-export-pcp2spark pcp-export-pcp2xml pcp-export-pcp2zabbix
571+
Requires: pcp-geolocate pcp-export-pcp2openmetrics pcp-export-pcp2opentelemetry
572+
Requires: pcp-export-pcp2json pcp-export-pcp2spark pcp-export-pcp2xml pcp-export-pcp2zabbix
573573
Requires: pcp-pmda-gluster pcp-pmda-zswap pcp-pmda-unbound pcp-pmda-mic
574574
Requires: pcp-pmda-libvirt pcp-pmda-lio pcp-pmda-openmetrics pcp-pmda-haproxy
575575
Requires: pcp-pmda-lmsensors pcp-pmda-netcheck pcp-pmda-rabbitmq pcp-pmda-uwsgi
@@ -878,6 +878,24 @@ Requires: %{__python2}-pcp = %{version}-%{release}
878878
Performance Co-Pilot (PCP) front-end tools for exporting metric values
879879
in OpenMetrics (https://openmetrics.io/) format.
880880

881+
#
882+
# pcp-export-pcp2opentelemetry
883+
#
884+
%package export-pcp2opentelemetry
885+
License: GPL-2.0-or-later
886+
Summary: Performance Co-Pilot tools for exporting PCP metrics in OpenTelemetry format
887+
URL: https://pcp.io
888+
Requires: pcp-libs >= %{version}-%{release}
889+
%if !%{disable_python3}
890+
Requires: python3-pcp = %{version}-%{release}
891+
%else
892+
Requires: %{__python2}-pcp = %{version}-%{release}
893+
%endif
894+
895+
%description export-pcp2opentelemetry
896+
Performance Co-Pilot (PCP) front-end tools for exporting metric values
897+
in OpenTelemetry (https://opentelemetry.io/) format.
898+
881899
#
882900
# pcp-export-pcp2spark
883901
#
@@ -2593,6 +2611,7 @@ basic_manifest | keep 'pcp2xlsx' >pcp-export-pcp2xlsx-files
25932611
basic_manifest | keep 'pcp2graphite' >pcp-export-pcp2graphite-files
25942612
basic_manifest | keep 'pcp2json' >pcp-export-pcp2json-files
25952613
basic_manifest | keep 'pcp2openmetrics' >pcp-export-pcp2openmetrics-files
2614+
basic_manifest | keep 'pcp2opentelemetry' >pcp-export-pcp2opentelemetry-files
25962615
basic_manifest | keep 'pcp2spark' >pcp-export-pcp2spark-files
25972616
basic_manifest | keep 'pcp2xml' >pcp-export-pcp2xml-files
25982617
basic_manifest | keep 'pcp2zabbix' >pcp-export-pcp2zabbix-files
@@ -2712,7 +2731,8 @@ done
27122731

27132732
for export_package in \
27142733
pcp2arrow pcp2elasticsearch pcp2graphite pcp2influxdb pcp2json \
2715-
pcp2openmetrics pcp2spark pcp2xlsx pcp2xml pcp2zabbix zabbix-agent ; \
2734+
pcp2openmetrics pcp2spark pcp2xlsx pcp2xml pcp2opentelemetry \
2735+
pcp2zabbix zabbix-agent ; \
27162736
do \
27172737
export_packages="$export_packages pcp-export-$export_package"; \
27182738
done
@@ -3411,6 +3431,8 @@ fi
34113431

34123432
%files export-pcp2openmetrics -f pcp-export-pcp2openmetrics-files.rpm
34133433

3434+
%files export-pcp2opentelemetry -f pcp-export-pcp2opentelemetry-files.rpm
3435+
34143436
%files export-pcp2spark -f pcp-export-pcp2spark-files.rpm
34153437

34163438
%files export-pcp2xml -f pcp-export-pcp2xml-files.rpm

Diff for: ‎qa/1131

+18-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/sh
22
# PCP QA Test No. 1131
3-
# Exercise pcp2json and pcp2openmetrics.
3+
# Exercise pcp2json, pcp2openmetrics, and pcp2opentelemetry.
44
#
55
# Copyright (c) 2017 Red Hat.
66
#
@@ -52,6 +52,21 @@ _filter_pcp2openmetrics()
5252
| LC_COLLATE=POSIX sort
5353
}
5454

55+
_filter_pcp2opentelemetry()
56+
{
57+
tee -a $here/$seq.full \
58+
| col -b \
59+
| sed \
60+
-e '/\"asDouble\":/ s/[0-9][0-9]*/NCPU/' \
61+
-e 's/\"'$machineid'\"/MACHINEID/' \
62+
-e 's/\"'$hostname'\"/HOSTNAME/' \
63+
-e 's/\"'$domainid'\"/DOMAINID/' \
64+
-e 's/\"agent\": .*/\"agent\": \"'$OSTYPE'\"/' \
65+
-e 's/\"startTimeUnixNano\": .*/\"startTimeUnixNano\": TIME/' \
66+
-e 's/\"userid\": .*/\"userid\": USERID/' \
67+
-e 's/\"userid\": .*/\"userid\": GROUPID/'
68+
}
69+
5570
# real QA test starts here
5671
echo "---"
5772
pcp2json -a $A -H -I -z "" | _archive_filter
@@ -64,6 +79,8 @@ pcp2json -a $A -H -I -z -X -b GB -P 2 -F $tmp.outfile ""
6479
echo "---"
6580
pcp2openmetrics -s1 -H -z hinv.ncpu | _filter_pcp2openmetrics
6681
echo "---"
82+
pcp2opentelemetry -s1 -H -z hinv.ncpu | _filter_pcp2opentelemetry
83+
echo "---"
6784

6885

6986
cat $tmp.outfile | _archive_filter

Diff for: ‎qa/1131.out

+55-1
Original file line numberDiff line numberDiff line change
@@ -2269,10 +2269,64 @@ QA output created by 1131
22692269
---
22702270

22712271

2272-
# HELP hinv_ncpu number of CPUs in the system
2272+
# HELP hinv_ncpu b'number of CPUs in the system'
22732273
# PCP5 hinv_ncpu 60.0.32 u32 PM_INDOM_NULL discrete
22742274
# TYPE hinv_ncpu gauge
22752275
hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} NCPU
2276+
---
2277+
{
2278+
"resourceMetrics": [
2279+
{
2280+
"resource": {
2281+
"attributes": [
2282+
{
2283+
"os.type": "linux",
2284+
"service.name": "pcp",
2285+
"telemetry.sdk.language": "python",
2286+
"telemetry.sdk.name": "opentelemetry",
2287+
"telemetry.sdk.version": "1.24.0"
2288+
}
2289+
]
2290+
},
2291+
"scopeMetrics": [
2292+
{
2293+
"metrics": [
2294+
{
2295+
"description": "number of CPUs in the system",
2296+
"gauge": {
2297+
"dataPoints": [
2298+
{
2299+
"asDouble": NCPU,
2300+
"attributes": [
2301+
{
2302+
"agent": "linux-gnu"
2303+
"domainname": DOMAINID,
2304+
"groupid": 4209557,
2305+
"hostname": HOSTNAME,
2306+
"machineid": MACHINEID,
2307+
"semantics": "discrete",
2308+
"type": "u32",
2309+
"userid": GROUPID
2310+
}
2311+
],
2312+
"startTimeUnixNano": TIME
2313+
}
2314+
]
2315+
},
2316+
"name": "hinv.ncpu",
2317+
"unit": "none"
2318+
}
2319+
],
2320+
"scope": {
2321+
"name": "pcp.hinv",
2322+
"version": 1
2323+
}
2324+
}
2325+
]
2326+
}
2327+
]
2328+
}
2329+
22762330
---
22772331
{
22782332
"@pcp": {

Diff for: ‎qa/1589

+2-9
Original file line numberDiff line numberDiff line change
@@ -51,16 +51,8 @@ _filter_pcp2json_http()
5151
-e "s/\(\"value\"\): \""$cpus"\"/\1:NCPU/g" \
5252
-e 's/\(\"@timestamp\"\): \(.*\)/\1:DATE TIME/' \
5353
-e '/HTTP/d' \
54-
-e "s/^\(Host: localhost\):$port/\1:PORT/g" \
55-
-e 's/^\(Content-Length:\) [1-9][0-9]*/\1 SIZE/g' \
56-
-e 's/^\(User-Agent: python-requests\).*/\1 VERSION/g' \
57-
-e 's/^\(Date:\).*/\1 DATE/g' \
58-
-e 's/\(\"context\":\) [0-9][0-9]*/\1 CTXID/g' \
59-
-e '/^Accept-Encoding: /d' \
60-
-e '/^Connection: keep-alive/d' \
61-
-e '/ using stream socket$/d' \
6254
| LC_COLLATE=POSIX sort
63-
}
55+
} >> $tmp.python2.out
6456

6557
# real QA test starts here
6658
port=`_find_free_port`
@@ -73,6 +65,7 @@ pcp2json -s1 -ZUTC -u http://localhost:$port hinv.ncpu >$tmp.json.out 2>$tmp.jso
7365

7466
echo "pcp2json HTTP POST (sorted):"
7567
_filter_pcp2json_http <$tmp.python.out
68+
_check_http_filter <$tmp.python2.out
7669

7770
# terminate pythonserver.py now
7871
pmsignal $pid >/dev/null 2>&1

Diff for: ‎qa/1827

+4-13
Original file line numberDiff line numberDiff line change
@@ -43,19 +43,9 @@ _filter_pcp2openmetrics_http()
4343
-e '/groupid=/s/groupid="*'$MYGID'"*/groupid="GROUPID"/' \
4444
-e '/userid=/s/userid="*'$MYUID'"*/userid="USERID"/' \
4545
-e '/hostname=/ s/hostname="*'$hostname'"*/hostname="HOST"/' \
46-
-e 's/} [0-9][0-9]*$/} N (cpus)/' \
47-
-e '/HTTP/d' \
48-
-e "s/^\(Host: localhost\):$port/\1:PORT/g" \
49-
-e 's/^\(Content-Length:\) [1-9][0-9]*/\1 SIZE/g' \
50-
-e 's/^\(User-Agent: python-requests\).*/\1 VERSION/g' \
51-
-e 's/^\(Date:\).*/\1 DATE/g' \
52-
-e 's/\(\"context\":\) [0-9][0-9]*/\1 CTXID/g' \
53-
-e '/^Accept-Encoding: /d' \
54-
-e 's/\(\hostname=\): \""$hostname"\"/\1:HOST/g' \
55-
-e '/^Connection: keep-alive/d' \
56-
-e '/ using stream socket$/d' \
57-
| LC_COLLATE=POSIX sort
58-
}
46+
-e '/HTTP/d' \
47+
-e 's/} [0-9][0-9]*$/} N (cpus)/'
48+
} >> $tmp.python2.out
5949

6050
# real QA test starts here
6151
port=`_find_free_port`
@@ -68,6 +58,7 @@ pcp2openmetrics -s1 -u http://localhost:$port hinv.ncpu >$tmp.openmetrics.out 2>
6858

6959
echo "pcp2openmetrics HTTP POST (sorted):"
7060
_filter_pcp2openmetrics_http <$tmp.python.out
61+
_check_http_filter <$tmp.python2.out
7162

7263
# terminate pythonserver.py now
7364
pmsignal $pid >/dev/null 2>&1

Diff for: ‎qa/1827.out

+13-13
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,23 @@
11
QA output created by 1827
22
pcp2openmetrics invocation
33
pcp2openmetrics HTTP POST (sorted):
4+
INFO:root:Starting httpd...
45

6+
INFO:root:POST request,
7+
Path: /
8+
Headers:
9+
Host: localhost:PORT
10+
User-Agent: python-requests VERSION
11+
Accept: */*
12+
Content-Type: application/openmetrics-text
13+
Content-Length: SIZE
514

615

716

8-
9-
10-
# HELP hinv_ncpu number of CPUs in the system
17+
Body:
1118
# PCP5 hinv_ncpu 60.0.32 u32 PM_INDOM_NULL discrete
19+
# HELP hinv_ncpu b'number of CPUs in the system'
1220
# TYPE hinv_ncpu gauge
13-
Accept: */*
14-
Body:
15-
Content-Length: SIZE
16-
Content-Type: application/openmetrics-text
17-
Headers:
18-
Host: localhost:PORT
19-
INFO:root:POST request,
20-
INFO:root:Starting httpd...
21-
Path: /
22-
User-Agent: python-requests VERSION
2321
hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} N (cpus)
22+
23+

Diff for: ‎qa/1977

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#!/bin/sh
2+
# PCP QA Test No. 1827
3+
# Exercise pcp2opentelemetry HTTP POST functionality.
4+
#
5+
# Copyright (c) 2024 Red Hat. All Rights Reserved.
6+
#
7+
8+
9+
seq=`basename $0`
10+
echo "QA output created by $seq"
11+
12+
. ./common.python
13+
14+
which pcp2opentelemetry >/dev/null 2>&1 || _notrun "pcp2opentelemetry not installed"
15+
16+
17+
_cleanup()
18+
{
19+
cd $here
20+
$sudo rm -rf $tmp $tmp.*
21+
}
22+
23+
status=0 # success is the default!
24+
cpus=`pmprobe -v hinv.ncpu | awk '{print $3}'`
25+
hostname=`hostname`
26+
machineid=`_machine_id`
27+
domainid=`_domain_name`
28+
$sudo rm -rf $tmp $tmp.* $seq.full
29+
trap "_cleanup; exit \$status" 0 1 2 3 15
30+
31+
_filter_pcp2opentelemetry_http()
32+
{
33+
tee -a $here/$seq.full \
34+
| col -b \
35+
| sed \
36+
-e '/\"asDouble\":/ s/[0-9][0-9]*/NCPU/' \
37+
-e 's/\"'$machineid'\"/MACHINEID/' \
38+
-e 's/\"'$hostname'\"/HOSTNAME/' \
39+
-e 's/\"'$domainid'\"/DOMAINID/' \
40+
-e 's/\"agent\": .*/\"agent\": \"'$OSTYPE'\"/' \
41+
-e 's/\"startTimeUnixNano\": .*/\"startTimeUnixNano\": TIME/' \
42+
-e 's/\"userid\": .*/\"userid\": USERID/' \
43+
-e 's/\"userid\": .*/\"userid\": GROUPID/' \
44+
-e '/HTTP/d'
45+
} >> $tmp.python2.out
46+
47+
48+
# real QA test starts here
49+
port=`_find_free_port`
50+
$PCP_PYTHON_PROG $here/src/pythonserver.py $port >$tmp.python.out 2>&1 &
51+
pid=$!
52+
sleep 2 # let server start up
53+
54+
55+
echo "pcp2opentelemetry invocation" | tee -a $here/$seq.full
56+
pcp2opentelemetry -s1 -u http://localhost:$port/receive hinv.ncpu >$tmp.json.out 2>$tmp.openjson.err
57+
58+
59+
echo "pcp2opentelemetry HTTP POST (sorted):"
60+
_filter_pcp2opentelemetry_http <$tmp.python.out
61+
_check_http_filter <$tmp.python2.out
62+
63+
64+
# terminate pythonserver.py now
65+
pmsignal $pid >/dev/null 2>&1
66+
67+
# success, all done
68+
exit

0 commit comments

Comments
 (0)