WIP: add RHOAI Notebooks Performance tests to the HOME page

backend/app/api/api.py

@@ -4,10 +4,12 @@
 from app.api.v1.endpoints.ocp import ocpJobs
 from app.api.v1.endpoints.ocp import graph
 from app.api.v1.endpoints.cpt import cptJobs
-from app.api.v1.endpoints.jira import jira
+#from app.api.v1.endpoints.jira import jira
 from app.api.v1.endpoints.quay import quayJobs
 from app.api.v1.endpoints.quay import quayGraphs
 
+from app.api.v1.endpoints.rhoai import jobs as rhoaiJobs
+
 router = APIRouter()
 
 # OCP endpoints
@@ -23,4 +25,7 @@
 router.include_router(quayGraphs.router, tags=['quay'])
 
 # Jira endpoints
-router.include_router(jira.router, tags=['jira'])
+#router.include_router(jira.router, tags=['jira'])
+
+# RHOAI endpoints
+router.include_router(rhoaiJobs.router, tags=['rhoai'])

backend/app/api/v1/commons/rhoai.py

@@ -0,0 +1,46 @@
+from datetime import date
+import pandas as pd
+from app.services.search import ElasticService
+
+
+import asyncio
+import typing
+
+
+def return_awaited_value(coroutine: asyncio.coroutines) -> typing.Any:
+
+    loop = asyncio.get_event_loop()
+    result = loop.run_until_complete(coroutine)
+    return result
+
+
+async def getData(start_datetime: date, end_datetime: date, configpath: str):
+    query = {
+        "timeout": "300s",
+        "query": {
+            "bool": {
+                "filter": {
+                    "range": {
+                        "metadata.start": {
+                            "format": "yyyy-MM-dd"
+                        }
+                    }
+                }
+            }
+        }
+    }
+    query['query']['bool']['filter']['range']['metadata.start']['lte'] = str(end_datetime)
+    query['query']['bool']['filter']['range']['metadata.start']['gte'] = str(start_datetime)
+
+    es = ElasticService(configpath=configpath)
+
+    response = await es.post(query)
+    await es.close()
+
+    tasks = [item['_source'] for item in response["hits"]["hits"]]
+    jobs = pd.json_normalize(tasks)
+
+    if len(jobs) == 0:
+        return jobs
+
+    return jobs

backend/app/api/v1/endpoints/cpt/cptJobs.py

@@ -6,15 +6,17 @@
 from .maps.ocp import ocpMapper
 from .maps.quay import quayMapper
 from .maps.hce import hceMapper
+from .maps.rhoai import rhoaiNotebooksPerformanceMapper
 from ...commons.example_responses import cpt_200_response, response_422
 from fastapi.param_functions import Query
 
 router = APIRouter()
 
 products = {
-            "ocp": ocpMapper,
-            "quay": quayMapper,
-            "hce": hceMapper
+            #"ocp": ocpMapper,
+            #"quay": quayMapper,
+            #"hce": hceMapper,
+            "rhoai": rhoaiNotebooksPerformanceMapper,
            }
 
 @router.get('/api/v1/cpt/jobs',
@@ -47,8 +49,9 @@ async def jobs(start_date: date = Query(None, description="Start date for search
             results = pd.concat([results, df.loc[:, ["ciSystem", "uuid", "releaseStream", "jobStatus", "buildUrl", "startDate", "endDate", "product", "version", "testName"]]])
         except ConnectionError:
             print("Connection Error in mapper for product " + product)
-        except:
-            print("Date range returned no values or Unknown error in mapper for product " + product)
+        except Exception as e: # DANGEROUS
+            print("Date range returned no values or Unknown error in mapper for product " + product + " | " + str(e))
+            raise e
 
     response = {
         'startDate': start_date.__str__(),

backend/app/api/v1/endpoints/cpt/maps/rhoai.py

@@ -0,0 +1,26 @@
+from ....commons.rhoai import getData
+from datetime import date
+
+
+################################################################
+# This will return a DataFrame required by the CPT endpoint
+################################################################
+
+async def rhoaiNotebooksPerformanceMapper(start_datetime: date, end_datetime: date, configpath: str):
+    df = await getData(start_datetime, end_datetime, configpath)
+    df.insert(len(df.columns), "product", "RHOAI Notebooks Performance")
+    df.insert(len(df.columns), "ciSystem", "Middleware Jenkins")
+    df.insert(len(df.columns), "releaseStream", "rc")
+    df.insert(len(df.columns), "buildUrl", "N/A")
+    df.insert(len(df.columns), "jobStatus", "success")
+
+    if df.empty:
+        return df
+
+    df["startDate"] = df["metadata.start"]
+    df["endDate"] = df["metadata.start"]
+    df["uuid"] = df["metadata.start"]
+    df["version"] = df["metadata.settings.version"].fillna("no version available")
+    df["testName"] = df["metadata.settings.image"].fillna("no test")
+
+    return df

backend/app/api/v1/endpoints/rhoai/jobs.py

@@ -0,0 +1,64 @@
+import json
+from fastapi import Response
+from datetime import datetime, timedelta, date
+from fastapi import APIRouter
+from ...commons.rhoai import getData as rhoai_getData
+from ...commons.example_responses import ocp_200_response, response_422
+from fastapi.param_functions import Query
+
+router = APIRouter()
+
+
+
+@router.get('/api/v1/rhoai/jobs',
+            summary="Returns a job list",
+            description="Returns a list of jobs in the specified dates. \
+            If not dates are provided the API will default the values. \
+            `startDate`: will be set to the day of the request minus 5 days.\
+            `endDate`: will be set to the day of the request.",
+            responses={
+                200: ocp_200_response(),
+                422: response_422(),
+        },)
+async def jobs(start_date: date = Query(None, description="Start date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-10"]),
+                end_date: date = Query(None, description="End date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-15"]),
+                pretty: bool = Query(False, description="Output contet in pretty format.")):
+
+    if start_date is None:
+        start_date = datetime.utcnow().date()
+        start_date = start_date - timedelta(days=50)
+
+    if end_date is None:
+        end_date = datetime.utcnow().date()
+
+    if start_date > end_date:
+        return Response(content=json.dumps({'error': "invalid date format, start_date must be less than end_date"}), status_code=422)
+
+    results = await rhoai_getData(start_date, end_date, 'rhoai.elasticsearch')
+
+    if len(results) >= 1 :
+        response = {
+            'startDate': start_date.__str__(),
+            'endDate': end_date.__str__(),
+            'results': results.to_dict('records')
+        }
+    else :
+        response = {
+            'startDate': start_date.__str__(),
+            'endDate': end_date.__str__(),
+            'results': []
+        }
+
+    for result in response["results"]:
+        result["benchmark"] = result["results.benchmark_measures.benchmark"]
+        result["shortVersion"] = result["metadata.rhods_version"]
+        result["platform"] = "OCP "+result["metadata.ocp_version"]
+        result["workers"] = "1"
+        result["ciSystem"] = "Middleware Jenkins"
+        result["networkType"] = "default"
+        result["jobType"] = "On demand"
+        result["jobStatus"] = "PASS"
+
+    json_str = json.dumps(response, indent=4 if pretty else None)
+
+    return Response(content=json_str, media_type='application/json')

backend/app/services/search.py

@@ -1,8 +1,10 @@
-from elasticsearch import AsyncElasticsearch
+from elasticsearch import AsyncElasticsearch, Elasticsearch
 from fastapi.encoders import jsonable_encoder
 
 from app import config
 
+import urllib3
+urllib3.disable_warnings()
 
 class ElasticService:
     # todo add bulkhead pattern
@@ -27,6 +29,12 @@ def __init__(self,configpath="",index=""):
                     verify_certs=False,
                     http_auth=(self.esUser,self.esPass)
             )
+            self.sync_es = Elasticsearch(
+                    self.url,
+                    use_ssl=False,
+                    verify_certs=False,
+                    http_auth=(self.esUser,self.esPass)
+            )
         else:
             self.es = AsyncElasticsearch(self.url, verify_certs=False)
 
@@ -36,7 +44,15 @@ async def post(self, query, indice=None,size=10000):
         return await self.es.search(
             index=indice,
             body=jsonable_encoder(query),
-            size=size)
+            size=size, request_timeout=60)
+
+    def sync_post(self, query, indice=None,size=10000):
+        if indice is None:
+            indice = self.indice
+        return self.sync_es.search(
+            index=indice,
+            body=jsonable_encoder(query),
+            size=size, request_timeout=60)
 
     async def close(self):
         await self.es.close()

backend/backend.containerfile

@@ -1,6 +1,8 @@
 FROM quay.io/centos/centos:stream9
 
-ENV PATH=/root/.local/bin:$PATH \
+ENV HOME=/backend
+
+ENV PATH=$HOME/.local/bin:$PATH \
     LANG=C.UTF-8 \
     PYTHONPATH=/backend
 
@@ -23,6 +25,7 @@ RUN pip install --user poetry && \
     pip install --user dash && \
     pip install --no-cache-dir -r requirements.txt
 
-ENTRYPOINT ["/bin/bash", "./scripts/start.sh"]
-
+RUN touch ocpperf.yaml && chmod 777 ocpperf.yaml
+RUN chgrp -R 0 $HOME && chmod -R g=u $HOME
 
+ENTRYPOINT ["/bin/bash", "./scripts/start.sh"]

backend/to_run

@@ -0,0 +1,7 @@
+podman run \
+    --interactive \
+    --tty \
+    --volume "$PWD/app:/backend/app:z" \
+    --volume "$PWD/ocpperf.toml:/backend/ocpperf.toml:z" \
+    --publish 8000:8000 \
+    ocpp-back /backend/scripts/start-reload.sh

frontend/frontend.containerfile

@@ -16,5 +16,7 @@ EXPOSE 3000
 RUN chown -R node /app
 USER node
 
+RUN mkdir -p /app/node_modules/.cache && chmod 777 /app/node_modules/.cache
+
 # start app
 CMD ["yarn", "run", "start"]

frontend/src/App.js

@@ -6,7 +6,7 @@ import {
     PageSection,
     PageSectionVariants,
 } from '@patternfly/react-core';
-import {fetchOCPJobsData, fetchCPTJobsData, fetchQuayJobsData} from "./store/Actions/ActionCreator";
+import {fetchOCPJobsData, fetchCPTJobsData, fetchQuayJobsData, fetchRHOAIJobsData} from "./store/Actions/ActionCreator";
 import {useDispatch} from "react-redux";
 import {Route, Switch, BrowserRouter as Router} from "react-router-dom";
 import {NavBar} from "./components/NavBar/NavBar";
@@ -20,9 +20,10 @@ export const App = () => {
 
     useEffect(() => {
           const fetchData = async () =>{
-              await dispatch(fetchOCPJobsData())
-              await dispatch(fetchCPTJobsData())
-              await dispatch(fetchQuayJobsData())
+              //await dispatch(fetchOCPJobsData())
+              //await dispatch(fetchCPTJobsData())
+              //await dispatch(fetchQuayJobsData())
+              await dispatch(fetchRHOAIJobsData())
           }
           fetchData()
     }, [dispatch])

frontend/src/components/templates/HomeLayout/DisplayTableDataLayout.js

@@ -13,7 +13,7 @@ export const DisplayTableDataLayout = ({initialState, tableMetaData, tableData,
        console.log(tableData)
         return  tableData && tableData.map( items => {
             const tableRows = tableMetaData.map( metadata => {
-                console.log(items[metadata.value])
+                //console.log(items[metadata.value])
                 if(metadata.name === 'Status')
                     if(items[metadata.value].toLowerCase() === "success")
                         return <Badge style={{backgroundColor: '#008000'}} children={items[metadata.value]} />

frontend/src/store/Actions/ActionCreator.js

@@ -1,5 +1,5 @@
 
-import {BASE_URL, OCP_GRAPH_API_V1, OCP_JOBS_API_V1, CPT_JOBS_API_V1, QUAY_JOBS_API_V1, QUAY_GRAPH_API_V1} from "../Shared";
+import {BASE_URL, OCP_GRAPH_API_V1, OCP_JOBS_API_V1, CPT_JOBS_API_V1, QUAY_JOBS_API_V1, QUAY_GRAPH_API_V1, RHOAI_API_V1} from "../Shared";
 import axios from "axios";
 import {
     errorOCPCall,
@@ -261,4 +261,53 @@ const GetComputeArchs = (api_data) => {
 
 const GetControlPlaneArchs = (api_data) => {
     return Array.from(new Set(api_data.map(item => item.controlPlaneArch.toUpperCase().trim()))).sort()
-}
+}
+
+export const fetchRHOAIJobsData = (startDate = '', endDate='') => async dispatch => {
+    let buildUrl = `${BASE_URL}${RHOAI_API_V1}`
+
+    dispatch(setWaitForOCPUpdate({waitForUpdate:true}))
+
+    if (startDate !== '' && endDate !== '') {
+        buildUrl += `&start_date=${startDate}&end_date=${endDate}`
+    }
+
+    try {
+        let api_data = await fetchAPI(buildUrl)
+        dispatch(setWaitForOCPUpdate({waitForUpdate:false}))
+
+        if (api_data){
+            const results = api_data.results
+            if (results){
+
+                const benchmarks = GetBenchmarks(results)
+                const versions = GetVersions(results)
+                const platforms = GetPlatforms(results)
+                const workers = GetWorkers(results)
+                const networkTypes = GetNetworkTypes(results)
+                const ciSystems = GetCiSystems(results)
+                const jobTypes = GetJobType(results)
+                const rehearses = ["TRUE", "FALSE"]
+                const updatedTime = new Date().toLocaleString().replace(', ', ' ').toString();
+
+                await dispatch(getOCPJobsData({
+                    data: results, benchmarks, versions, waitForUpdate: false, platforms, workers, networkTypes,
+                    updatedTime, ciSystems, jobTypes, rehearses, startDate: api_data.startDate, endDate: api_data.endDate
+                }))
+
+                await dispatch(updateOCPMetaData({data: results}))
+            }
+        }
+    }
+    catch (e) {
+        const error = e
+        if(error.response){
+            await dispatch(errorOCPCall({error: error.response.data.error}))
+            alert(error.response.data.error)
+        }
+        else{
+            console.log(error)
+        }
+        dispatch(setWaitForOCPUpdate({waitForUpdate:false}))
+    }
+}

frontend/src/store/Shared.js

@@ -13,4 +13,6 @@ export const OCP_GRAPH_API_V1 = "/api/v1/ocp/graph"
 export const CPT_JOBS_API_V1 = "/api/v1/cpt/jobs"
 
 export const QUAY_JOBS_API_V1 = "/api/v1/quay/jobs"
-export const QUAY_GRAPH_API_V1 = "/api/v1/quay/graph"
+export const QUAY_GRAPH_API_V1 = "/api/v1/quay/graph"
+
+export const RHOAI_API_V1 = "/api/v1/rhoai/jobs"