From a21ace030a9aaa397e2ca09ec85819acbce88b90 Mon Sep 17 00:00:00 2001 From: rafabailon Date: Fri, 14 Jun 2024 13:05:08 +0200 Subject: [PATCH 01/10] feat: add method to unify process values --- .../wazuh_testing/scripts/data_visualizations.py | 4 +++- .../tools/performance/visualization.py | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py b/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py index f7c665dc4f..d43e27641d 100644 --- a/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py +++ b/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py @@ -20,6 +20,8 @@ def get_script_arguments(): help=f'Base name for the images. Default {None}.') parser.add_argument('-c', '--columns', dest='columns', default=None, help=f'Path to Json with Columns to Plot. Default {None}.') + parser.add_argument('-u', '--unify', dest='unify', default=False, + help=f'Unify process values. Default {False}.') return parser.parse_args() @@ -32,7 +34,7 @@ def main(): makedirs(destination) dv = DataVisualizer(dataframes=options.csv_list, target=options.visualization_target, compare=False, store_path=options.destination, base_name=options.name, - columns_path=options.columns) + columns_path=options.columns, unify=options.unify) dv.plot() diff --git a/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py b/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py index 807d3ed5b5..fdf5eb8c16 100644 --- a/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py +++ b/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py @@ -32,7 +32,7 @@ class DataVisualizer: base_name (str, optional): base name used to store the images. """ def __init__(self, dataframes, target, compare=False, store_path=gettempdir(), x_ticks_granularity='minutes', - x_ticks_interval=1, base_name=None, columns_path=None): + x_ticks_interval=1, base_name=None, columns_path=None, unify=False): self.dataframes_paths = dataframes self.dataframe = None self.compare = compare @@ -48,6 +48,9 @@ def __init__(self, dataframes, target, compare=False, store_path=gettempdir(), x if target in ['binary', 'analysis', 'remote', 'agent', 'logcollector', 'wazuhdb']: self.columns_to_plot = self._load_columns_to_plot(columns_path) + if unify: + self._unify_dataframes() + @staticmethod def _color_palette(size): """Create a list of different colors. @@ -87,6 +90,17 @@ def _load_dataframes(self): new_csv = pd.read_csv(df_path, index_col="Timestamp", parse_dates=True) self.dataframe = pd.concat([self.dataframe, new_csv]) + def _unify_dataframes(self): + """Unify dataframe values.""" + df_row = self.dataframe.iloc[0] + df_names = [df_row['Daemon'], df_row['Version'], df_row['PID']] + columns_to_drop = ['Daemon', 'Version', 'PID'] + columns_to_sum = self.dataframe.columns.drop(columns_to_drop) + self.dataframe = self.dataframe.groupby('Timestamp')[columns_to_sum].sum().reset_index(drop=False) + + for index, value in enumerate(df_names): + self.dataframe.insert(index, columns_to_drop[index], value) + def _set_x_ticks_interval(self, ax): """Set the number of labels that will appear in the X axis and their format. From 854660f33b7f8e75466499f59fc79a4e4d2803f0 Mon Sep 17 00:00:00 2001 From: rafabailon Date: Fri, 14 Jun 2024 16:22:28 +0200 Subject: [PATCH 02/10] feat: modify conditional --- .../wazuh_testing/tools/performance/visualization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py b/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py index fdf5eb8c16..f102a7689e 100644 --- a/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py +++ b/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py @@ -48,7 +48,7 @@ def __init__(self, dataframes, target, compare=False, store_path=gettempdir(), x if target in ['binary', 'analysis', 'remote', 'agent', 'logcollector', 'wazuhdb']: self.columns_to_plot = self._load_columns_to_plot(columns_path) - if unify: + if unify.lower() in ["true"]: self._unify_dataframes() @staticmethod From 01333c0e6591812f0e7f03bc2c8a99ab7f0de728 Mon Sep 17 00:00:00 2001 From: rafabailon Date: Fri, 14 Jun 2024 16:44:04 +0200 Subject: [PATCH 03/10] feat: fix linter errors --- deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py b/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py index d43e27641d..80cd8e8168 100644 --- a/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py +++ b/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py @@ -21,7 +21,7 @@ def get_script_arguments(): parser.add_argument('-c', '--columns', dest='columns', default=None, help=f'Path to Json with Columns to Plot. Default {None}.') parser.add_argument('-u', '--unify', dest='unify', default=False, - help=f'Unify process values. Default {False}.') + help=f'Unify process values. Default {False}.') return parser.parse_args() From 4f128638fdbf2976fae270c15f42731b033bddf2 Mon Sep 17 00:00:00 2001 From: rafabailon Date: Tue, 18 Jun 2024 08:47:41 +0200 Subject: [PATCH 04/10] feat: rename unify variable --- .../wazuh_testing/scripts/data_visualizations.py | 2 +- .../wazuh_testing/tools/performance/visualization.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py b/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py index 80cd8e8168..cc9d3d7929 100644 --- a/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py +++ b/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py @@ -34,7 +34,7 @@ def main(): makedirs(destination) dv = DataVisualizer(dataframes=options.csv_list, target=options.visualization_target, compare=False, store_path=options.destination, base_name=options.name, - columns_path=options.columns, unify=options.unify) + columns_path=options.columns, unify_child_daemon_metrics=options.unify) dv.plot() diff --git a/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py b/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py index f102a7689e..9b28ef8c5f 100644 --- a/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py +++ b/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py @@ -32,7 +32,7 @@ class DataVisualizer: base_name (str, optional): base name used to store the images. """ def __init__(self, dataframes, target, compare=False, store_path=gettempdir(), x_ticks_granularity='minutes', - x_ticks_interval=1, base_name=None, columns_path=None, unify=False): + x_ticks_interval=1, base_name=None, columns_path=None, unify_child_daemon_metrics=False): self.dataframes_paths = dataframes self.dataframe = None self.compare = compare @@ -48,7 +48,7 @@ def __init__(self, dataframes, target, compare=False, store_path=gettempdir(), x if target in ['binary', 'analysis', 'remote', 'agent', 'logcollector', 'wazuhdb']: self.columns_to_plot = self._load_columns_to_plot(columns_path) - if unify.lower() in ["true"]: + if unify_child_daemon_metrics.lower() in ["true"]: self._unify_dataframes() @staticmethod From 0c04723e4abb3dc501fa02aa43816985a0e8edd5 Mon Sep 17 00:00:00 2001 From: rafabailon Date: Tue, 18 Jun 2024 08:49:35 +0200 Subject: [PATCH 05/10] feat: change in the conditional to unify --- .../wazuh_testing/tools/performance/visualization.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py b/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py index 9b28ef8c5f..52d1f5611a 100644 --- a/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py +++ b/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py @@ -48,7 +48,8 @@ def __init__(self, dataframes, target, compare=False, store_path=gettempdir(), x if target in ['binary', 'analysis', 'remote', 'agent', 'logcollector', 'wazuhdb']: self.columns_to_plot = self._load_columns_to_plot(columns_path) - if unify_child_daemon_metrics.lower() in ["true"]: + if unify_child_daemon_metrics.lower() in ["true"] and target in ['binary']: + self.dataframe = self.dataframe.reset_index(drop=False) self._unify_dataframes() @staticmethod From 76c2ea93317f9f7d702788cb92faacf0d2386309 Mon Sep 17 00:00:00 2001 From: rafabailon Date: Tue, 18 Jun 2024 08:52:31 +0200 Subject: [PATCH 06/10] feat: modify the logic of unification --- .../tools/performance/visualization.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py b/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py index 52d1f5611a..b788c495c1 100644 --- a/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py +++ b/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py @@ -93,14 +93,19 @@ def _load_dataframes(self): def _unify_dataframes(self): """Unify dataframe values.""" + daemons_list = [daemon_name for daemon_name in self._get_daemons() if "child" not in daemon_name] + + for daemon_name in daemons_list: + self.dataframe.loc[self.dataframe['Daemon'].str.contains(daemon_name, na=False), 'Daemon'] = daemon_name + df_row = self.dataframe.iloc[0] - df_names = [df_row['Daemon'], df_row['Version'], df_row['PID']] - columns_to_drop = ['Daemon', 'Version', 'PID'] + df_names = [df_row['Version'], df_row['PID']] + columns_to_drop = ['Timestamp', 'Daemon', 'Version', 'PID'] columns_to_sum = self.dataframe.columns.drop(columns_to_drop) - self.dataframe = self.dataframe.groupby('Timestamp')[columns_to_sum].sum().reset_index(drop=False) + self.dataframe = self.dataframe.groupby(['Timestamp', 'Daemon'])[columns_to_sum].sum().reset_index(drop=False) for index, value in enumerate(df_names): - self.dataframe.insert(index, columns_to_drop[index], value) + self.dataframe.insert(index+2, columns_to_drop[index+2], value) def _set_x_ticks_interval(self, ax): """Set the number of labels that will appear in the X axis and their format. From 91ad2fe6bb18a484c011f6c3bedf8ce32f9e8609 Mon Sep 17 00:00:00 2001 From: rafabailon Date: Tue, 18 Jun 2024 10:33:06 +0200 Subject: [PATCH 07/10] feat: Improvement of the unification logic --- .../wazuh_testing/tools/performance/visualization.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py b/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py index b788c495c1..85acb60fdc 100644 --- a/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py +++ b/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py @@ -93,19 +93,21 @@ def _load_dataframes(self): def _unify_dataframes(self): """Unify dataframe values.""" + pids = self.dataframe[['Daemon', 'PID']].drop_duplicates() + versions = self.dataframe[['Daemon', 'Version']].drop_duplicates() + daemons_list = [daemon_name for daemon_name in self._get_daemons() if "child" not in daemon_name] for daemon_name in daemons_list: self.dataframe.loc[self.dataframe['Daemon'].str.contains(daemon_name, na=False), 'Daemon'] = daemon_name - df_row = self.dataframe.iloc[0] - df_names = [df_row['Version'], df_row['PID']] columns_to_drop = ['Timestamp', 'Daemon', 'Version', 'PID'] columns_to_sum = self.dataframe.columns.drop(columns_to_drop) + self.dataframe = self.dataframe.groupby(['Timestamp', 'Daemon'])[columns_to_sum].sum().reset_index(drop=False) - for index, value in enumerate(df_names): - self.dataframe.insert(index+2, columns_to_drop[index+2], value) + self.dataframe = self.dataframe.merge(pids[['Daemon', 'PID']], on='Daemon', how='left') + self.dataframe = self.dataframe.merge(versions[['Daemon', 'Version']], on='Daemon', how='left') def _set_x_ticks_interval(self, ax): """Set the number of labels that will appear in the X axis and their format. From 5757bb6e4daaad5a08a8d592b6da57dbbe97de17 Mon Sep 17 00:00:00 2001 From: rafabailon Date: Wed, 19 Jun 2024 12:57:25 +0200 Subject: [PATCH 08/10] feat: change the argument information to unify --- deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py b/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py index cc9d3d7929..895f768f09 100644 --- a/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py +++ b/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py @@ -21,7 +21,7 @@ def get_script_arguments(): parser.add_argument('-c', '--columns', dest='columns', default=None, help=f'Path to Json with Columns to Plot. Default {None}.') parser.add_argument('-u', '--unify', dest='unify', default=False, - help=f'Unify process values. Default {False}.') + help=f'Unify data of the binary processes with their subprocesses to plot. Default {False}.') return parser.parse_args() From ee85a1dc18d61450298897d04af637de0c4f2051 Mon Sep 17 00:00:00 2001 From: rafabailon Date: Wed, 19 Jun 2024 14:52:22 +0200 Subject: [PATCH 09/10] feat: change conditional, argument and docstring --- .../wazuh_testing/scripts/data_visualizations.py | 4 ++-- .../tools/performance/visualization.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py b/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py index 895f768f09..8ba897af7b 100644 --- a/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py +++ b/deps/wazuh_testing/wazuh_testing/scripts/data_visualizations.py @@ -20,8 +20,8 @@ def get_script_arguments(): help=f'Base name for the images. Default {None}.') parser.add_argument('-c', '--columns', dest='columns', default=None, help=f'Path to Json with Columns to Plot. Default {None}.') - parser.add_argument('-u', '--unify', dest='unify', default=False, - help=f'Unify data of the binary processes with their subprocesses to plot. Default {False}.') + parser.add_argument('-u', '--unify', dest='unify', action='store_true', + help=f'Unify data of the binary processes with their subprocesses to plot.') return parser.parse_args() diff --git a/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py b/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py index 85acb60fdc..31ead4be57 100644 --- a/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py +++ b/deps/wazuh_testing/wazuh_testing/tools/performance/visualization.py @@ -4,6 +4,7 @@ from matplotlib.ticker import LinearLocator import json +import logging import matplotlib.dates as mdates import matplotlib.pyplot as plt import pandas as pd @@ -48,9 +49,12 @@ def __init__(self, dataframes, target, compare=False, store_path=gettempdir(), x if target in ['binary', 'analysis', 'remote', 'agent', 'logcollector', 'wazuhdb']: self.columns_to_plot = self._load_columns_to_plot(columns_path) - if unify_child_daemon_metrics.lower() in ["true"] and target in ['binary']: - self.dataframe = self.dataframe.reset_index(drop=False) - self._unify_dataframes() + if unify_child_daemon_metrics: + if target == 'binary': + self.dataframe = self.dataframe.reset_index(drop=False) + self._unify_dataframes() + else: + logging.warning("Enabled unify is only available for binary data. Ignoring") @staticmethod def _color_palette(size): @@ -92,7 +96,8 @@ def _load_dataframes(self): self.dataframe = pd.concat([self.dataframe, new_csv]) def _unify_dataframes(self): - """Unify dataframe values.""" + """Unify the data of each process with their respective sub-processes. + """ pids = self.dataframe[['Daemon', 'PID']].drop_duplicates() versions = self.dataframe[['Daemon', 'Version']].drop_duplicates() From 90d45d9efe1beda78a7de4ca5e81ddfa5a2ecd30 Mon Sep 17 00:00:00 2001 From: rafabailon Date: Thu, 20 Jun 2024 17:30:42 +0200 Subject: [PATCH 10/10] feat: update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c67e0c56d..d565137521 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. ## [4.8.1] - TBD +### Added + +- Add functionality to unify data of the binary processes with their subprocesses to plot ([#5500](https://github.com/wazuh/wazuh-qa/pull/5500)) \- (Framework) + ## [4.8.0] - 12/06/2024 ### Added