From 22d672af15b79ecc841339704d91dd1cfd8f50bf Mon Sep 17 00:00:00 2001 From: vishnuchalla Date: Wed, 22 May 2024 18:27:06 -0400 Subject: [PATCH 1/2] Adding plotly visuals for telco tests Signed-off-by: vishnuchalla --- backend/app/api/api.py | 2 + .../app/api/v1/endpoints/telco/telcoGraphs.py | 388 ++++++++++++++++++ frontend/package.json | 2 +- .../ReactGraphs/plotly/PlotlyView.js | 10 +- .../src/components/Telco/BenchmarkResults.js | 27 +- frontend/src/components/Telco/DisplayGraph.js | 85 ++-- frontend/src/store/Actions/ActionCreator.js | 21 +- frontend/src/store/Shared.js | 3 +- frontend/src/store/reducers/InitialData.js | 7 +- .../src/store/reducers/TelcoGraphReducer.js | 24 ++ frontend/src/store/reducers/index.js | 2 + 11 files changed, 521 insertions(+), 50 deletions(-) create mode 100644 backend/app/api/v1/endpoints/telco/telcoGraphs.py create mode 100644 frontend/src/store/reducers/TelcoGraphReducer.js diff --git a/backend/app/api/api.py b/backend/app/api/api.py index 01473cee..48743aed 100644 --- a/backend/app/api/api.py +++ b/backend/app/api/api.py @@ -8,6 +8,7 @@ from app.api.v1.endpoints.quay import quayJobs from app.api.v1.endpoints.quay import quayGraphs from app.api.v1.endpoints.telco import telcoJobs +from app.api.v1.endpoints.telco import telcoGraphs router = APIRouter() @@ -25,6 +26,7 @@ # Telco endpoints router.include_router(telcoJobs.router, tags=['telco']) +router.include_router(telcoGraphs.router, tags=['telco']) # Jira endpoints router.include_router(jira.router, tags=['jira']) \ No newline at end of file diff --git a/backend/app/api/v1/endpoints/telco/telcoGraphs.py b/backend/app/api/v1/endpoints/telco/telcoGraphs.py new file mode 100644 index 00000000..e4761534 --- /dev/null +++ b/backend/app/api/v1/endpoints/telco/telcoGraphs.py @@ -0,0 +1,388 @@ +from fastapi import APIRouter +import app.api.v1.commons.hasher as hasher + +router = APIRouter() + +@router.get("/api/v1/telco/graph/{uuid}/{encryptedData}") +async def graph(uuid: str, encryptedData: str): + bytesData = encryptedData.encode("utf-8") + decrypted_data = hasher.decrypt_unhash_json(uuid, bytesData) + json_data = decrypted_data["data"] + return await process_json(json_data) + +async def process_json(json_data: dict): + function_mapper = { + "ptp": process_ptp, + "oslat": process_oslat, + "reboot": process_reboot, + "cpu_util": process_cpu_util, + "rfc-2544": process_rfc_2544, + "cyclictest": process_cyclictest, + "deployment": process_deployment, + } + mapped_function = function_mapper.get(json_data["test_type"]) + return mapped_function(json_data) + +def process_ptp(json_data: str): + nic = json_data["nic"] + ptp4l_max_offset = json_data["ptp4l_max_offset"] + if "mellanox" in nic.lower(): + defined_offset_threshold = 200 + else: + defined_offset_threshold = 100 + minus_offset = 0 + if ptp4l_max_offset > defined_offset_threshold: + minus_offset = ptp4l_max_offset - defined_offset_threshold + + return { + "ptp": [ + { + "name": "Data Points", + "x": ["ptp4l_max_offset"], + "y": [ptp4l_max_offset], + "mode": "markers", + "marker": { + "size": 10, + }, + "error_y": { + "type": "data", + "symmetric": "false", + "array": [0], + "arrayminus": [minus_offset] + }, + + }, + { + "name": "Threshold", + "x": ["ptp4l_max_offset"], + "y": [defined_offset_threshold], + "mode": "lines+markers", + "line": { + "dash": 'dot', + "width": 3, + }, + "marker": { + "size": 15, + }, + "type": "scatter", + } + ] + } + + +def process_reboot(json_data: str): + max_minutes = 0.0 + avg_minutes = 0.0 + minus_max_minutes = 0.0 + minus_avg_minutes = 0.0 + defined_threshold = 20 + reboot_type = json_data["reboot_type"] + for each_iteration in json_data["Iterations"]: + max_minutes = max(max_minutes, each_iteration["total_minutes"]) + avg_minutes += each_iteration["total_minutes"] + avg_minutes /= len(json_data["Iterations"]) + if max_minutes > defined_threshold: + minus_max_minutes = max_minutes - defined_threshold + if avg_minutes > defined_threshold: + minus_avg_minutes = avg_minutes - defined_threshold + + return { + "reboot": [ + { + "name": "Data Points", + "x": [reboot_type + "_" + "max_minutes", reboot_type + "_" + "avg_minutes"], + "y": [max_minutes, avg_minutes], + "mode": "markers", + "marker": { + "size": 10, + }, + "error_y": { + "type": "data", + "symmetric": "false", + "array": [0, 0], + "arrayminus": [minus_max_minutes, minus_avg_minutes] + }, + "type": "scatter", + }, + { + "name": "Threshold", + "x": [reboot_type + "_" + "max_minutes", reboot_type + "_" + "avg_minutes"], + "y": [defined_threshold, defined_threshold], + "mode": "lines+markers", + "marker": { + "size": 15, + }, + "line": { + "dash": "dot", + "width": 3, + }, + "type": "scatter", + } + ] + } + +def process_cpu_util(json_data: str): + total_max_cpu = 0.0 + total_avg_cpu = 0.0 + minus_max_cpu = 0.0 + minus_avg_cpu = 0.0 + defined_threshold = 3.0 + for each_scenario in json_data["scenarios"]: + if each_scenario["scenario_name"] == "steadyworkload": + for each_type in each_scenario["types"]: + if each_type["type_name"] == "total": + total_max_cpu = each_type["max_cpu"] + break + total_avg_cpu = each_scenario["avg_cpu_total"] + break + if total_max_cpu > defined_threshold: + minus_max_cpu = total_max_cpu - defined_threshold + if total_avg_cpu > defined_threshold: + minus_avg_cpu = total_avg_cpu - defined_threshold + + return { + "cpu_util": [ + { + "name": "Data Points", + "x": ["total_max_cpu", "total_avg_cpu"], + "y": [total_max_cpu, total_avg_cpu], + "mode": "markers", + "marker": { + "size": 10, + }, + "error_y": { + "type": "data", + "symmetric": "false", + "array": [0, 0], + "arrayminus": [minus_max_cpu, minus_avg_cpu] + }, + "type": "scatter", + }, + { + "name": "Threshold", + "x": ["total_max_cpu", "total_avg_cpu"], + "y": [defined_threshold, defined_threshold], + "mode": "lines+markers", + "marker": { + "size": 15, + }, + "line": { + "dash": "dot", + "width": 3, + }, + "type": "scatter", + } + ] + } + +def process_rfc_2544(json_data: str): + max_delay = json_data["max_delay"] + defined_delay_threshold = 30.0 + minus_max_delay = 0.0 + if max_delay > defined_delay_threshold: + minus_max_delay = max_delay - defined_delay_threshold + + return { + "rfc-2544": [ + { + "x": ["max_delay"], + "y": [max_delay], + "mode": "markers", + "marker": { + "size": 10, + }, + "name": "Data Points", + "error_y": { + "type": "data", + "symmetric": "false", + "array": [0], + "arrayminus": [minus_max_delay] + }, + "type": "scatter", + }, + { + "x": ["max_delay"], + "y": [defined_delay_threshold], + "name": "Threshold", + "mode": "lines+markers", + "marker": { + "size": 15, + }, + "line": { + "dash": "dot", + "width": 3, + }, + "type": "scatter" + } + ] + } + +def process_oslat(json_data: str): + return { + "oslat": get_oslat_or_cyclictest(json_data) + } + +def process_cyclictest(json_data: str): + return { + "cyclictest": get_oslat_or_cyclictest(json_data) + } + +def process_deployment(json_data: str): + total_minutes = json_data["total_minutes"] + reboot_count = json_data["reboot_count"] + defined_total_minutes_threshold = 180 + defined_total_reboot_count = 3 + minus_total_minutes = 0.0 + minus_total_reboot_count = 0.0 + if total_minutes > defined_total_minutes_threshold: + minus_total_minutes = total_minutes - defined_total_minutes_threshold + if reboot_count > defined_total_reboot_count: + minus_total_reboot_count = reboot_count - defined_total_reboot_count + + return { + "deployment": { + "total_minutes": [ + { + "name": "Data Points", + "x": ["total_minutes"], + "y": [total_minutes], + "mode": "markers", + "marker": { + "size": 10, + }, + "error_y": { + "type": "data", + "symmetric": "false", + "array": [0], + "arrayminus": [minus_total_minutes] + }, + "type": "scatter", + }, + { + "name": "Threshold", + "x": ["total_minutes"], + "y": [defined_total_minutes_threshold], + "mode": "lines+markers", + "marker": { + "size": 15, + }, + "line": { + "dash": "dot", + "width": 3, + }, + "type": "scatter", + } + ], + "total_reboot_count": [ + { + "name": "Data Points", + "x": ["reboot_count"], + "y": [reboot_count], + "mode": "markers", + "marker": { + "size": 10, + }, + "error_y": { + "type": "data", + "symmetric": "false", + "array": [0], + "arrayminus": [minus_total_reboot_count] + }, + "type": "scatter", + }, + { + "name": "Threshold", + "x": ["reboot_count"], + "y": [defined_total_reboot_count], + "mode": "lines+markers", + "marker": { + "size": 15, + }, + "line": { + "dash": "dot", + "width": 3, + }, + "type": "scatter", + } + ] + } + } + +def get_oslat_or_cyclictest(json_data: str): + min_number_of_nines = 10000 + max_latency = 0 + minus_max_latency = 0 + defined_latency_threshold = 20 + defined_number_of_nines_threshold = 100 + for each_test_unit in json_data["test_units"]: + max_latency = max(max_latency, each_test_unit["max_latency"]) + min_number_of_nines = min(min_number_of_nines, each_test_unit["number_of_nines"]) + if max_latency > defined_latency_threshold: + minus_max_latency = max_latency - defined_latency_threshold + + return { + "number_of_nines": [ + { + "name": "Data Points", + "x": ["min_number_of_nines"], + "y": [min_number_of_nines], + "mode": "markers", + "marker": { + "size": 10, + }, + "error_y": { + "type": "data", + "symmetric": "false", + "array": [0], + "arrayminus": [min_number_of_nines - defined_number_of_nines_threshold] + }, + "type": "scatter", + }, + { + "name": "Threshold", + "x": ["min_number_of_nines"], + "y": [defined_number_of_nines_threshold], + "mode": "lines+markers", + "marker": { + "size": 15, + }, + "line": { + "dash": "dot", + "width": 3, + }, + "type": "scatter", + } + ], + "max_latency": [ + { + "name": "Data Points", + "x": ["max_latency"], + "y": [max_latency], + "mode": "markers", + "marker": { + "size": 10, + }, + "error_y": { + "type": "data", + "symmetric": "false", + "array": [0], + "arrayminus": [minus_max_latency] + }, + "type": "scatter", + }, + { + "name": "Threshold", + "x": ["max_latency"], + "y": [defined_latency_threshold], + "mode": "lines+markers", + "marker": { + "size": 15, + }, + "line": { + "dash": "dot", + "width": 3, + }, + "type": "scatter", + } + ] + } diff --git a/frontend/package.json b/frontend/package.json index 085e7c1c..073e105b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -11,7 +11,7 @@ "@testing-library/react": "^11.1.0", "@testing-library/user-event": "^12.1.10", "axios": "^1.5.0", - "plotly.js": "^2.26.0", + "plotly.js": "^2.32.0", "prop-types": "^15.8.1", "react": "^17.0.1", "react-dom": "^17.0.1", diff --git a/frontend/src/components/ReactGraphs/plotly/PlotlyView.js b/frontend/src/components/ReactGraphs/plotly/PlotlyView.js index e3f51e52..a4641d17 100644 --- a/frontend/src/components/ReactGraphs/plotly/PlotlyView.js +++ b/frontend/src/components/ReactGraphs/plotly/PlotlyView.js @@ -1,11 +1,11 @@ -import Plotly from "react-plotly.js"; +import Plot from "react-plotly.js"; import React from "react"; export const PlotlyView = ({data, width = "100%", height = "100%"}) => { - return } diff --git a/frontend/src/components/Telco/BenchmarkResults.js b/frontend/src/components/Telco/BenchmarkResults.js index b2692dd7..bbe1d975 100644 --- a/frontend/src/components/Telco/BenchmarkResults.js +++ b/frontend/src/components/Telco/BenchmarkResults.js @@ -6,16 +6,23 @@ import {DisplayGraph} from "./DisplayGraph"; export const BenchmarkResults = ({dataset, isExpanded}) => { return ( - <> { - ( isExpanded && - - - - - - ) || <>NO Data - } + <> { + (isExpanded && + + + + + + + + + ) || <>NO Data + } ) } diff --git a/frontend/src/components/Telco/DisplayGraph.js b/frontend/src/components/Telco/DisplayGraph.js index a9e18244..000e4d7f 100644 --- a/frontend/src/components/Telco/DisplayGraph.js +++ b/frontend/src/components/Telco/DisplayGraph.js @@ -1,38 +1,61 @@ -import {PlotlyView} from "../ReactGraphs/plotly/PlotlyView"; -import React, {useEffect} from "react"; -import {useDispatch, useSelector} from "react-redux"; +import { PlotlyView } from "../ReactGraphs/plotly/PlotlyView"; +import React, { useEffect, useState } from "react"; +import { useDispatch, useSelector } from "react-redux"; import CardView from "../PatternflyComponents/Card/CardView"; -import {Text6} from "../PatternflyComponents/Text/Text"; -import {SplitView} from "../PatternflyComponents/Split/SplitView"; -import {Spinner} from "@patternfly/react-core"; -import {fetchGraphData} from "../../store/Actions/ActionCreator"; +import { Text6 } from "../PatternflyComponents/Text/Text"; +import { SplitView } from "../PatternflyComponents/Split/SplitView"; +import { Spinner } from "@patternfly/react-core"; +import { fetchTelcoGraphData } from "../../store/Actions/ActionCreator"; +export const DisplayGraph = ({ uuid, encryptedData, benchmark, heading }) => { + const [isExpanded, setExpanded] = useState(true); + const onExpand = () => setExpanded(!isExpanded); -export const DisplayGraph = ({uuid, benchmark}) => { + const dispatch = useDispatch(); + const jobResults = useSelector(state => state.telcoGraph); - const [isExpanded, setExpanded] = React.useState(true) - const onExpand = () => setExpanded(!isExpanded) + useEffect(() => { + dispatch(fetchTelcoGraphData(uuid, encryptedData)); + }, [dispatch, uuid, encryptedData]); - const dispatch = useDispatch() - const job_results = useSelector(state => state.graph) - const graphData = job_results.uuid_results[uuid] + const graphData = jobResults.uuid_results[uuid]; - useEffect(() => { - dispatch(fetchGraphData(uuid)) - }, [dispatch, uuid]) - - const getGraphBody = () => { - return (job_results.graphError && ) || - (graphData && ) || - , ]} /> - } - - return <> - } - body={ getGraphBody() } - isExpanded={isExpanded} - expandView={true} - onExpand={onExpand} - /> + const getGraphBody = (key = null) => { + const benchmarkGraph = graphData && graphData[benchmark]; + const dataForKey = key === null ? benchmarkGraph : benchmarkGraph?.[key]; + + return jobResults.graphError + ? + : dataForKey + ? + : , ]} />; + }; + + const renderCard = (key, customHeading) => ( + } + body={getGraphBody(key)} + isExpanded={isExpanded} + expandView={true} + onExpand={onExpand} + /> + ); + + return ( + <> + {benchmark === 'oslat' || benchmark === 'cyclictest' ? ( + <> + {renderCard('number_of_nines', `${benchmark} results`)} + {renderCard('max_latency', `${benchmark} results`)} + + ) : benchmark === 'deployment' ? ( + <> + {renderCard('total_minutes', `${benchmark} results`)} + {renderCard('total_reboot_count', `${benchmark} results`)} + + ) : ( + renderCard(null, `${benchmark} results`) + )} -} + ); +}; diff --git a/frontend/src/store/Actions/ActionCreator.js b/frontend/src/store/Actions/ActionCreator.js index 938d914e..54af73bb 100644 --- a/frontend/src/store/Actions/ActionCreator.js +++ b/frontend/src/store/Actions/ActionCreator.js @@ -1,5 +1,5 @@ -import {BASE_URL, OCP_GRAPH_API_V1, OCP_JOBS_API_V1, CPT_JOBS_API_V1, QUAY_JOBS_API_V1, QUAY_GRAPH_API_V1, TELCO_JOBS_API_V1} from "../Shared"; +import {BASE_URL, OCP_GRAPH_API_V1, OCP_JOBS_API_V1, CPT_JOBS_API_V1, QUAY_JOBS_API_V1, QUAY_GRAPH_API_V1, TELCO_JOBS_API_V1, TELCO_GRAPH_API_V1} from "../Shared"; import axios from "axios"; import { errorOCPCall, @@ -27,6 +27,7 @@ import { } from "../reducers/TelcoJobsReducer"; import {getUuidResults, setGraphError} from "../reducers/GraphReducer"; import {getQuayUuidResults, setQuayGraphError} from "../reducers/QuayGraphReducer"; +import {getTelcoUuidResults, setTelcoGraphError} from "../reducers/TelcoGraphReducer"; export const fetchAPI = async (url, requestOptions = {}) => { const response = await axios(url, requestOptions) @@ -69,6 +70,24 @@ export const fetchQuayGraphData = (uuid) => async dispatch =>{ } } +export const fetchTelcoGraphData = (uuid, encryptedData) => async dispatch =>{ + try { + let buildUrl = `${BASE_URL}${TELCO_GRAPH_API_V1}/${uuid}/${encryptedData}` + const api_data = await fetchAPI(buildUrl) + if(api_data) dispatch(getTelcoUuidResults({ [uuid]: api_data })) + } + catch (error){ + if (axios.isAxiosError(error)) { + console.error('Axios Error:', error); + console.error('Request:', error.request); + console.error('Response:', error.response); + } else { + console.error('Axios Error:', error); + dispatch(setTelcoGraphError({error: error.response.data.details})) + } + } +} + export const fetchOCPJobsData = (startDate = '', endDate='') => async dispatch => { let buildUrl = `${BASE_URL}${OCP_JOBS_API_V1}` dispatch(setWaitForOCPUpdate({waitForUpdate:true})) diff --git a/frontend/src/store/Shared.js b/frontend/src/store/Shared.js index fd2856f5..cff99f6a 100644 --- a/frontend/src/store/Shared.js +++ b/frontend/src/store/Shared.js @@ -15,4 +15,5 @@ export const CPT_JOBS_API_V1 = "/api/v1/cpt/jobs" export const QUAY_JOBS_API_V1 = "/api/v1/quay/jobs" export const QUAY_GRAPH_API_V1 = "/api/v1/quay/graph" -export const TELCO_JOBS_API_V1 = "/api/v1/telco/jobs" \ No newline at end of file +export const TELCO_JOBS_API_V1 = "/api/v1/telco/jobs" +export const TELCO_GRAPH_API_V1 = "/api/v1/telco/graph" \ No newline at end of file diff --git a/frontend/src/store/reducers/InitialData.js b/frontend/src/store/reducers/InitialData.js index e60bbfbe..75ef869b 100644 --- a/frontend/src/store/reducers/InitialData.js +++ b/frontend/src/store/reducers/InitialData.js @@ -171,4 +171,9 @@ export const GRAPH_INITIAL_DATA = { export const QUAY_GRAPH_INITIAL_DATA = { uuid_results: {}, graphError: false, -} \ No newline at end of file +} + +export const TELCO_GRAPH_INITIAL_DATA = { + uuid_results: {}, + graphError: false, +} diff --git a/frontend/src/store/reducers/TelcoGraphReducer.js b/frontend/src/store/reducers/TelcoGraphReducer.js new file mode 100644 index 00000000..934eb18d --- /dev/null +++ b/frontend/src/store/reducers/TelcoGraphReducer.js @@ -0,0 +1,24 @@ +import {createSlice} from "@reduxjs/toolkit"; +import {TELCO_GRAPH_INITIAL_DATA} from "./InitialData"; + + +const telcoGraphReducer = createSlice({ + initialState: { + ...TELCO_GRAPH_INITIAL_DATA, + }, + name: 'telcoGraph', + reducers: { + getTelcoUuidResults: (state, action) => { + Object.assign(state.uuid_results, action.payload) + }, + setTelcoGraphError: (state, action) => { + Object.assign(state.graphError, action.payload.error) + } + } +}) + +export const { + getTelcoUuidResults, + setTelcoGraphError +} = telcoGraphReducer.actions +export default telcoGraphReducer.reducer \ No newline at end of file diff --git a/frontend/src/store/reducers/index.js b/frontend/src/store/reducers/index.js index ea1b6f90..fe4fddad 100644 --- a/frontend/src/store/reducers/index.js +++ b/frontend/src/store/reducers/index.js @@ -4,6 +4,7 @@ import quayJobsReducer from "./QuayJobsReducer"; import telcoJobsReducer from "./TelcoJobsReducer"; import graphReducer from "./GraphReducer"; import quayGraphReducer from "./QuayGraphReducer"; +import telcoGraphReducer from "./TelcoGraphReducer"; export const rootReducer = { @@ -13,4 +14,5 @@ export const rootReducer = { 'telcoJobs': telcoJobsReducer, 'graph': graphReducer, 'quayGraph': quayGraphReducer, + 'telcoGraph': telcoGraphReducer, } From 75f424b955c14aac8bd3f0dbbb9e196bb044c4a2 Mon Sep 17 00:00:00 2001 From: vishnuchalla Date: Thu, 23 May 2024 14:50:02 -0400 Subject: [PATCH 2/2] Fixing jenkins url Signed-off-by: vishnuchalla --- backend/app/api/v1/commons/telco.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app/api/v1/commons/telco.py b/backend/app/api/v1/commons/telco.py index 49a055cd..49656b9a 100644 --- a/backend/app/api/v1/commons/telco.py +++ b/backend/app/api/v1/commons/telco.py @@ -57,7 +57,7 @@ async def getData(start_datetime: date, end_datetime: date, configpath: str): 'formal': test_data['formal'], "startDate": str(start_time_utc), "endDate": str(end_time_utc), - "buildUrl": jenkins_url + str(test_data['cluster_artifacts']['ref']['jenkins_build']), + "buildUrl": jenkins_url + "/" + str(test_data['cluster_artifacts']['ref']['jenkins_build']), "jobStatus": "success", "jobDuration": execution_time_seconds, })