performancecopilot · natoscott · Apr 9, 2025 · Apr 14, 2025 · Apr 15, 2025 · Apr 15, 2025
diff --git a/build/containers/archive-analysis/root/usr/bin/archive-import.py b/build/containers/archive-analysis/root/usr/bin/archive-import.py
@@ -1,4 +1,19 @@
-#!/usr/bin/env python3
+#!/usr/bin/env pmpython
+#
+# Copyright (C) 2025 Marko Myllynen <[email protected]>
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+
+# pylint: disable=global-statement
+
 import logging
 import time
 import argparse

diff --git a/build/rpm/pcp.spec.in b/build/rpm/pcp.spec.in
@@ -459,6 +459,22 @@ Performance Co-Pilot (PCP) front-end tools for importing ganglia data
 into standard PCP archive logs for replay with any PCP monitoring tool.
 %endif
 
+%if "@enable_python3@" == "true"
+#
+# pcp-import-pmseries
+#
+%package import-pmseries
+License: LGPL-2.1-or-later
+Summary: Performance Co-Pilot tools importing PCP archives for pmseries queries
+URL: https://pcp.io
+Requires: pcp-libs = @package_version@
+Requires: python3-pcp = @package_version@
+
+%description import-pmseries
+Performance Co-Pilot (PCP) tools for importing PCP archives into Valkey
+or Redis for fast, scalable time series access via pmseries(1) queries.
+%endif
+
 #
 # pcp-import-collectl2pcp
 #
@@ -2332,6 +2348,7 @@ basic_manifest | grep -E -e 'pmiostat|pmrep|dstat|htop|pcp2csv' \
    -e 'pcp-tapestat|pcp-uptime|pcp-verify|pcp-xsos' | \
    cull 'selinux|pmlogconf|pmieconf|pmrepconf' >pcp-system-tools-files
 basic_manifest | keep 'geolocate' >pcp-geolocate-files
+basic_manifest | keep 'pmseries_import' >pcp-import-pmseries-files
 basic_manifest | keep 'sar2pcp' >pcp-import-sar2pcp-files
 basic_manifest | keep 'iostat2pcp' >pcp-import-iostat2pcp-files
 basic_manifest | keep 'sheet2pcp' >pcp-import-sheet2pcp-files
@@ -2457,6 +2474,7 @@ do \
 done
 
 for import_package in \
+    pmseries \
     collectl2pcp iostat2pcp ganglia2pcp mrtg2pcp sar2pcp sheet2pcp ; \
 do \
     import_packages="$import_packages pcp-import-$import_package"; \
@@ -3287,6 +3305,10 @@ fi
 
 %files pmda-weblog -f pcp-pmda-weblog-files.rpm
 
+%if "@enable_python3@" == "true"
+%files import-pmseries -f pcp-import-pmseries-files.rpm
+%endif
+
 %if "@have_perl@" == "true"
 %files import-sar2pcp -f pcp-import-sar2pcp-files.rpm
 

diff --git a/build/rpm/redhat.spec b/build/rpm/redhat.spec
@@ -731,6 +731,22 @@ Requires: pcp-libs = %{version}-%{release}
 Performance Co-Pilot (PCP) module for exporting metrics from PCP to
 Zabbix via the Zabbix agent - see zbxpcp(3) for further details.
 
+%if !%{disable_python3}
+#
+# pcp-import-pmseries
+#
+%package import-pmseries
+License: LGPL-2.1-or-later
+Summary: Performance Co-Pilot tools importing PCP archives for pmseries queries
+URL: https://pcp.io
+Requires: pcp-libs = @package_version@
+Requires: python3-pcp = @package_version@
+
+%description import-pmseries
+Performance Co-Pilot (PCP) tools for importing PCP archives into Valkey
+or Redis for fast, scalable time series access via pmseries(1) queries.
+%endif
+
 %if !%{disable_python2} || !%{disable_python3}
 #
 # pcp-geolocate
@@ -2570,6 +2586,7 @@ basic_manifest | grep -E -e 'pmiostat|pmrep|dstat|htop|pcp2csv' \
    -e 'pcp-tapestat|pcp-uptime|pcp-verify|pcp-xsos' | \
    cull 'selinux|pmlogconf|pmieconf|pmrepconf' >pcp-system-tools-files
 basic_manifest | keep 'geolocate' >pcp-geolocate-files
+basic_manifest | keep 'pmseries_import' >pcp-import-pmseries-files
 basic_manifest | keep 'sar2pcp' >pcp-import-sar2pcp-files
 basic_manifest | keep 'iostat2pcp' >pcp-import-iostat2pcp-files
 basic_manifest | keep 'sheet2pcp' >pcp-import-sheet2pcp-files
@@ -2696,6 +2713,7 @@ do \
 done
 
 for import_package in \
+    pmseries \
     collectl2pcp iostat2pcp ganglia2pcp mrtg2pcp sar2pcp sheet2pcp ; \
 do \
     import_packages="$import_packages pcp-import-$import_package"; \
@@ -3502,6 +3520,10 @@ fi
 
 %files pmda-weblog -f pcp-pmda-weblog-files.rpm
 
+%if !%{disable_python3}
+%files import-pmseries -f pcp-import-pmseries-files.rpm
+%endif
+
 %if !%{disable_perl}
 %files import-sar2pcp -f pcp-import-sar2pcp-files.rpm
 

diff --git a/man/man1/pmseries_import.1 b/man/man1/pmseries_import.1
@@ -0,0 +1,62 @@
+'\"macro stdmacro
+.\"
+.\" Copyright (c) 2025 Red Hat.
+.\"
+.\" This program is free software; you can redistribute it and/or modify it
+.\" under the terms of the GNU General Public License as published by the
+.\" Free Software Foundation; either version 2 of the License, or (at your
+.\" option) any later version.
+.\"
+.\" This program is distributed in the hope that it will be useful, but
+.\" WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+.\" or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+.\" for more details.
+.\"
+.TH PMSERIES_IMPORT 1 "PCP" "Performance Co-Pilot"
+.SH NAME
+\f3pmseries_import\f1 \- import PCP archive files for historical time series querying
+.SH SYNOPSIS
+.B $PCP_BINADM_DIR/pmseries_import
+[\f3\--archives\f1 \f2path\f1]
+.SH DESCRIPTION
+.B pmseries_import
+monitors a given \f2path\f1 for PCP archives, and automatically
+loads the contents into a key server (such as Valkey or Redis).
+This is then suitable for fast time series querying using the
+.BR pmseries (1)
+language, as provided by the command line tool and PCP REST API
+services of
+.BR pmproxy (1).
+.PP
+.B pmseries_import
+uses a simple polling technique, checking the archives directory
+for new archive files every 10 seconds.
+.PP
+This utility is most commonly used within a container providing
+an automated Grafana dashboard setup for arbitrary PCP archives.
+This approach provides an environment containing a
+.BR systemd (1)
+server to enable this archive monitoring utility, pre-configured
+.BR grafana-server (1),
+.BR valkey-server (1),
+and
+.BR pmproxy (1)
+REST API services.
+.SH OPTIONS
+.TP 5
+\fB\--archives\fR=\fIpath\fR
+Specifies an alternate
+.I path
+to poll for new PCP archives.
+Default value is
+.IR /archives .
+.SH SEE ALSO
+.BR grafana-server (1),
+.BR systemd (1),
+.BR valkey-server (1),
+.BR pmproxy (1)
+and
+.BR pmseries (1).
+
+.\" control lines for scripts/man-spell
+.\" +ok+ grafana Grafana pre valkey Valkey
diff --git a/qa/1807 b/qa/1807
@@ -0,0 +1,65 @@
+#!/bin/sh
+# PCP QA Test No. 1807
+# Exercise the pmseries_import script.
+#
+# Copyright (c) 2025 Red Hat.  All Rights Reserved.
+#
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+. ./common.python
+. ./common.keys
+
+test -x $PCP_BINADM_DIR/pmseries_import || _notrun "No pmseries_import script"
+_check_series
+
+_cleanup()
+{
+    [ -n "$key_server_port" ] && $keys_cli -p $key_server_port shutdown
+    _restore_config $PCP_SYSCONF_DIR/pmseries
+    cd $here
+    $sudo rm -rf $tmp $tmp.*
+}
+
+status=0	# success is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_filter()
+{
+    sed \
+	-e "s@$here@PATH@g" \
+	-e "s@$tmp@TMP@g" \
+    # end
+}
+
+# real QA test starts here
+key_server_port=`_find_free_port`
+_save_config $PCP_SYSCONF_DIR/pmseries
+$sudo rm -f $PCP_SYSCONF_DIR/pmseries/*
+
+echo "Start test key server ..."
+$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 &
+_check_key_server_ping $key_server_port
+_check_key_server $key_server_port
+echo
+
+_check_key_server_version $key_server_port
+
+mkdir $tmp/archives
+cp $here/archives/pcp-meminfo* $tmp/archives
+args="-p $key_server_port -Z UTC"
+
+echo "== Auto-load metric data into this key server instance"
+pmseries_import $args --archives $tmp/archives >$tmp.import 2>&1 &
+pid=$!
+pmsleep 10 # delay for polling and loading
+_filter_import < $tmp.import 
+echo
+
+echo "== Verify archive data has been loaded into key server"
+pmseries $args 'mem.util.free[count:5]'
+echo
+
+# success, all done
+exit
diff --git a/qa/1807.out b/qa/1807.out
@@ -0,0 +1,3 @@
+QA output created by 1807
+== Auto-load metric data into this key server instance
+== Verify archive data has been loaded into key server
diff --git a/qa/group b/qa/group
@@ -2159,6 +2159,7 @@ pmcd.pdu
 1801 dstat python pcp local derive
 1803 python geolocate labels local pmjson
 1805 pmda.linux kernel local
+1807 pmseries local
 1810 pmda.bpf local
 1813 python labels local pmrep
 1814 pmda.linux local

diff --git a/src/pmseries/.gitignore b/src/pmseries/.gitignore
@@ -1,4 +1,6 @@
 pmseries
+pmseries_import.py
+pmseries_import.service
 jsmn.c
 jsmn.h
 sha1.h

diff --git a/src/pmseries/GNUmakefile b/src/pmseries/GNUmakefile
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2017-2019 Red Hat.
+# Copyright (c) 2017-2019,2025 Red Hat.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -17,31 +17,54 @@ include $(TOPDIR)/src/include/builddefs
 
 CFILES = pmseries.c
 CMDTARGET = pmseries$(EXECSUFFIX)
+CMDIMPORT = pmseries_import
+CONTAINER = build/containers/archive-analysis/root/usr/bin/archive-import.py
+XFILES = $(CMDIMPORT).py
 LLDLIBS	= $(PCP_WEBLIB)
 LCFLAGS = $(LIBUVCFLAGS)
 ifeq "$(TARGET_OS)" "mingw"
 LLDLIBS += -lws2_32
 endif
+LDIRT += $(XFILES) $(CMDIMPORT).service
 BASHDIR = $(PCP_BASHSHARE_DIR)/completions
 
 default:	build-me
 
 ifeq "$(HAVE_LIBUV)" "true"
-build-me:	$(CMDTARGET)
+build-me:	$(CMDTARGET) $(XFILES) $(CMDIMPORT).service
 
 install:	default
 	$(INSTALL) -m 755 $(CMDTARGET) $(PCP_BIN_DIR)/$(CMDTARGET)
 	$(INSTALL) -m 755 -d $(PCP_SYSCONF_DIR)/pmseries
-	# compatibility symlink (share pmproxy config Redis server list)
+	# compatibility symlink (sharing pmproxy config key server list)
 	$(INSTALL) -S $(PCP_SYSCONF_DIR)/pmproxy/pmproxy.conf $(PCP_SYSCONF_DIR)/pmseries/pmseries.conf
 	$(INSTALL) -S $(BASHDIR)/pcp $(BASHDIR)/$(CMDTARGET)
+ifeq ($(ENABLE_PYTHON3),true)
+	$(INSTALL) -m 755 $(CMDIMPORT).py $(PCP_BINADM_DIR)/$(CMDIMPORT)
+ifeq ($(ENABLE_SYSTEMD),true)
+	$(INSTALL) -m 644 $(CMDIMPORT).service $(PCP_SYSTEMDUNIT_DIR)/$(CMDIMPORT).service
+endif
+endif
 else
 build-me:
 install:
 endif
 
 include $(BUILDRULES)
 
+# TODO: move this file here instead of symlinking, once transition completed
+$(XFILES):
+	$(LN_S) $(TOPDIR)/$(CONTAINER) $@
+
 default_pcp : default
 
 install_pcp : install
+
+$(CMDIMPORT).service: $(CMDIMPORT).service.in
+	$(SED) <$< >$@ \
+	    -e 's;@PCP_BINADM_DIR@;'$(PCP_BINADM_DIR)';' \
+	    -e 's;@PCP_RUN_DIR@;'$(PCP_RUN_DIR)';' \
+	# END
+
+check :: $(CMDIMPORT).py
+	$(PYLINT) $^
diff --git a/src/pmseries/pmseries_import.service.in b/src/pmseries/pmseries_import.service.in
@@ -0,0 +1,15 @@
+[Unit]
+Description=Performance Co-Pilot time series importing
+Documentation=man:pmseries_import(1)
+After=valkey.service redis.service
+
+[Service]
+Type=exec
+Restart=on-failure
+StandardOutput=journal+console
+Environment=PYTHONUNBUFFERED=1
+ExecStart=@PCP_BINADM_DIR@/pmseries_import
+SyslogIdentifier=pmseries
+
+[Install]
+WantedBy=multi-user.target