Quay onboarding initial draft

backend/app/api/api.py

@@ -4,6 +4,8 @@
 from app.api.v1.endpoints.ocp import ocpJobs
 from app.api.v1.endpoints.ocp import graph
 from app.api.v1.endpoints.cpt import cptJobs
+from app.api.v1.endpoints.quay import quayJobs
+from app.api.v1.endpoints.quay import quayGraphs
 
 router = APIRouter()
 
@@ -14,3 +16,7 @@
 
 # CPT endopoints
 router.include_router(cptJobs.router, tags=['cpt'])
+
+# Quay endpoints
+router.include_router(quayJobs.router, tags=['quay'])
+router.include_router(quayGraphs.router, tags=['quay'])

backend/app/api/v1/commons/hce.py

@@ -3,7 +3,7 @@
 from app.services.search import ElasticService
 
 
-async def getData(start_datetime: date, end_datetime: date):
+async def getData(start_datetime: date, end_datetime: date, configpath: str):
     query = {
         "query": {
             "bool": {
@@ -20,7 +20,7 @@ async def getData(start_datetime: date, end_datetime: date):
     query['query']['bool']['filter']['range']['date']['lte'] = str(end_datetime)
     query['query']['bool']['filter']['range']['date']['gte'] = str(start_datetime)
 
-    es = ElasticService(configpath="hce.elasticsearch")
+    es = ElasticService(configpath=configpath)
     response = await es.post(query)
     await es.close()
     tasks = [item['_source'] for item in response["hits"]["hits"]]

backend/app/api/v1/commons/ocp.py

@@ -3,7 +3,7 @@
 from app.services.search import ElasticService
 
 
-async def getData(start_datetime: date, end_datetime: date):
+async def getData(start_datetime: date, end_datetime: date, configpath: str):
     query = {
         "query": {
             "bool": {
@@ -20,7 +20,7 @@ async def getData(start_datetime: date, end_datetime: date):
     query['query']['bool']['filter']['range']['timestamp']['lte'] = str(end_datetime)
     query['query']['bool']['filter']['range']['timestamp']['gte'] = str(start_datetime)
 
-    es = ElasticService(configpath="ocp.elasticsearch")
+    es = ElasticService(configpath=configpath)
     response = await es.post(query)
     await es.close()
     tasks = [item['_source'] for item in response["hits"]["hits"]]
@@ -35,6 +35,7 @@ async def getData(start_datetime: date, end_datetime: date):
     jobs["benchmark"] = jobs.apply(updateBenchmark, axis=1)
     jobs["jobType"] = jobs.apply(jobType, axis=1)
     jobs["isRehearse"] = jobs.apply(isRehearse, axis=1)
+    jobs["jobStatus"] = jobs.apply(updateStatus, axis=1)
 
     cleanJobs = jobs[jobs['platform'] != ""]
 
@@ -44,6 +45,10 @@ async def getData(start_datetime: date, end_datetime: date):
     return jbs
 
 
+def updateStatus(row):
+    return row["jobStatus"].lower()
+
+
 def updateBenchmark(row):
     if row["upstreamJob"].__contains__("upgrade"):
         return "upgrade-" + row["benchmark"]

backend/app/api/v1/commons/utils.py

@@ -0,0 +1,17 @@
+from app.services.search import ElasticService
+
+async def getMetadata(uuid: str, configpath: str) :
+    query = {
+        "query": {
+            "query_string": {
+                "query": (
+                    f'uuid: "{uuid}"')
+            }
+        }
+    }
+    print(query)
+    es = ElasticService(configpath=configpath)
+    response = await es.post(query)
+    await es.close()
+    meta = [item['_source'] for item in response["hits"]["hits"]]
+    return meta[0]

backend/app/api/v1/endpoints/cpt/cptJobs.py

@@ -4,6 +4,7 @@
 from datetime import datetime, timedelta, date
 from fastapi import APIRouter
 from .maps.ocp import ocpMapper
+from .maps.quay import quayMapper
 from .maps.hce import hceMapper
 from ...commons.example_responses import cpt_200_response, response_422
 from fastapi.param_functions import Query
@@ -12,6 +13,7 @@
 
 products = {
             "ocp": ocpMapper,
+            "quay": quayMapper,
             "hce": hceMapper
            }
 
@@ -41,8 +43,8 @@ async def jobs(start_date: date = Query(None, description="Start date for search
     results = pd.DataFrame()
     for product in products:
         try:
-            df = await products[product](start_date, end_date)
-            results = pd.concat([results, df])
+            df = await products[product](start_date, end_date, f'{product}.elasticsearch')
+            results = pd.concat([results, df.loc[:, ["ciSystem", "uuid", "releaseStream", "jobStatus", "buildUrl", "startDate", "endDate", "product", "version", "testName"]]])
         except ConnectionError:
             print("Connection Error in mapper for product " + product)
         except:

backend/app/api/v1/endpoints/cpt/maps/hce.py

@@ -15,8 +15,8 @@
 #   "version"
 #   "testName"
 ################################################################
-async def hceMapper(start_datetime: date, end_datetime: date):
-    df = await getData(start_datetime, end_datetime)
+async def hceMapper(start_datetime: date, end_datetime: date, configpath: str):
+    df = await getData(start_datetime, end_datetime, configpath)
     df["releaseStream"] = "Nightly"
     df["ciSystem"] = "Jenkins"
     df["testName"] = df["product"] + ":" + df["test"]

backend/app/api/v1/endpoints/cpt/maps/ocp.py

@@ -3,35 +3,14 @@
 
 
 ################################################################
-# This will return a DataFrame from OCP required by the CPT
-#  endpoint, it contians the following columns:
-#   "ciSystem"
-#   "uuid"
-#   "releaseStream"
-#   "jobStatus"
-#   "buildUrl"
-#   "startDate"
-#   "endDate"
-#   "product"
-#   "version"
-#   "testName"
+# This will return a DataFrame from OCP required by the CPT endpoint
 ################################################################
-async def ocpMapper(start_datetime: date, end_datetime: date):
-    df = await getData(start_datetime, end_datetime)
+async def ocpMapper(start_datetime: date, end_datetime: date, configpath: str):
+    df = await getData(start_datetime, end_datetime, configpath)
     df.insert(len(df.columns), "product", "ocp")
     df["releaseStream"] = df.apply(getReleaseStream, axis=1)
     df["version"] = df["shortVersion"]
     df["testName"] = df["benchmark"]
-    df = dropColumns(df)
-    return df
-
-
-def dropColumns(df):
-    df = df.drop(columns=['shortVersion', 'benchmark', 'platform', 'clusterType', 'masterNodesCount',
-                          'workerNodesCount', 'infraNodesCount', 'masterNodesType', 'workerNodesType',
-                          'infraNodesType', 'totalNodesCount', 'clusterName', 'ocpVersion', 'networkType',
-                          'buildTag', 'upstreamJob', 'upstreamJobBuild', 'executionDate', 'jobDuration',
-                          'timestamp', 'jobType', 'isRehearse'])
     return df
 
 

backend/app/api/v1/endpoints/cpt/maps/quay.py

@@ -0,0 +1,19 @@
+from ....commons.ocp import getData
+from datetime import date
+
+
+################################################################
+# This will return a DataFrame from Quay required by the CPT endpoint
+################################################################
+async def quayMapper(start_datetime: date, end_datetime: date, configpath: str):
+    df = await getData(start_datetime, end_datetime, configpath)
+    df.insert(len(df.columns), "product", "quay")
+    df["releaseStream"] = df.apply(getReleaseStream, axis=1)
+    df["version"] = df["shortVersion"]
+    df["testName"] = df["benchmark"]
+    return df
+
+def getReleaseStream(row):
+    if row["releaseStream"].__contains__("nightly"):
+        return "Nightly"
+    return "Stable"

backend/app/api/v1/endpoints/ocp/graph.py

@@ -1,12 +1,12 @@
 from datetime import datetime,timedelta
+from app.api.v1.commons.utils import getMetadata
 import trio
 import semver
 from fastapi import APIRouter
 import io
 import pprint
 from json import loads
 import pandas as pd
-import datetime
 from app.services.search import ElasticService
 
 router = APIRouter()
@@ -112,7 +112,7 @@ def parseCPUResults(data: dict):
 @router.get('/api/ocp/v1/graph/{uuid}')
 async def graph(uuid: str):
     index = ""
-    meta = await getMetadata(uuid)
+    meta = await getMetadata(uuid, 'ocp.elasticsearch')
     print(meta)
     metrics = []
     if meta["benchmark"] == "k8s-netperf" :
@@ -419,23 +419,6 @@ async def getMatchRuns(meta: dict, workerCount: False):
         uuids.append(run["uuid"])
     return uuids
 
-async def getMetadata(uuid: str) :
-    index = "perf_scale_ci"
-    query = {
-        "query": {
-            "query_string": {
-                "query": (
-                    f'uuid: "{uuid}"')
-            }
-        }
-    }
-    print(query)
-    es = ElasticService(configpath="ocp.elasticsearch")
-    response = await es.post(query)
-    await es.close()
-    meta = [item['_source'] for item in response["hits"]["hits"]]
-    return meta[0]
-
 """
     [ {
         'y' : ["4,13","4.14"],

backend/app/api/v1/endpoints/ocp/ocpJobs.py

@@ -33,7 +33,7 @@ async def jobs(start_date: date = Query(None, description="Start date for search
     if start_date > end_date:
         return Response(content=json.dumps({'error': "invalid date format, start_date must be less than end_date"}), status_code=422)
 
-    results = await getData(start_date, end_date)
+    results = await getData(start_date, end_date, 'ocp.elasticsearch')
 
     response = {
         'startDate': start_date.__str__(),