PANDA-690

cloud-bulldozer · Dec 5, 2024 · c17e5db · c17e5db
1 parent a1e8c44
commit c17e5db
Show file tree

Hide file tree

Showing 30 changed files with 1,506 additions and 1,255 deletions.
diff --git a/backend/app/api/api.py b/backend/app/api/api.py
@@ -16,26 +16,26 @@
 router = APIRouter()
 
 # OCP endpoints
-router.include_router(ocpJobs.router, tags=['ocp'])
-router.include_router(results.router, tags=['ocp'])
-router.include_router(graph.router, tags=['ocp.graphs'])
+router.include_router(ocpJobs.router, tags=["ocp"])
+router.include_router(results.router, tags=["ocp"])
+router.include_router(graph.router, tags=["ocp.graphs"])
 
 # CPT endpoints
-router.include_router(cptJobs.router, tags=['cpt'])
+router.include_router(cptJobs.router, tags=["cpt"])
 
 # Quay endpoints
-router.include_router(quayJobs.router, tags=['quay'])
-router.include_router(quayGraphs.router, tags=['quay'])
+router.include_router(quayJobs.router, tags=["quay"])
+router.include_router(quayGraphs.router, tags=["quay"])
 
 # Telco endpoints
-router.include_router(telcoJobs.router, tags=['telco'])
-router.include_router(telcoGraphs.router, tags=['telco'])
+router.include_router(telcoJobs.router, tags=["telco"])
+router.include_router(telcoGraphs.router, tags=["telco"])
 
 # Jira endpoints
-router.include_router(jira.router, tags=['jira'])
+router.include_router(jira.router, tags=["jira"])
 
 # Horreum endpoint
-router.include_router(horreum.router, tags=['horreum'])
+router.include_router(horreum.router, tags=["horreum"])
 
 # OCM endpoint
-router.include_router(ocmJobs.router, tags=['ocm'])
+router.include_router(ocmJobs.router, tags=["ocm"])
diff --git a/backend/app/api/v1/commons/example_responses.py b/backend/app/api/v1/commons/example_responses.py
@@ -1,22 +1,26 @@
 def response_200(example):
     return {
-                "content": {
-                    "application/json": {
-                        "example": example,
-                    }
-                },
+        "content": {
+            "application/json": {
+                "example": example,
             }
+        },
+    }
+
 
 def response_422():
     return {
-                "content": {
-                    "application/json": {
-                        "example": {"error": "invalid date format, start_date must be less than end_date"},
-                    }
+        "content": {
+            "application/json": {
+                "example": {
+                    "error": "invalid date format, start_date must be less than end_date"
                 },
             }
+        },
+    }
 
-ocp_response_example ={
+
+ocp_response_example = {
     "startDate": "2023-09-20",
     "endDate": "2023-09-20",
     "results": [
@@ -47,7 +51,7 @@ def response_422():
             "startDate": "2023-09-20T02:14:07Z",
             "endDate": "2023-09-20T03:41:48Z",
             "timestamp": "2023-09-20T02:14:07Z",
-            "shortVersion": "4.14"
+            "shortVersion": "4.14",
         },
         {
             "ciSystem": "PROW",
@@ -76,12 +80,12 @@ def response_422():
             "startDate": "2023-09-20T07:19:00Z",
             "endDate": "2023-09-20T07:28:42Z",
             "timestamp": "2023-09-20T07:19:00Z",
-            "shortVersion": "4.13"
+            "shortVersion": "4.13",
         },
-    ]
+    ],
 }
 
-quay_response_example ={
+quay_response_example = {
     "startDate": "2023-09-20",
     "endDate": "2023-09-20",
     "results": [
@@ -114,12 +118,12 @@ def response_422():
             "startDate": "2023-09-20T02:14:07Z",
             "endDate": "2023-09-20T03:41:48Z",
             "timestamp": "2023-09-20T02:14:07Z",
-            "shortVersion": "4.14"
+            "shortVersion": "4.14",
         },
-    ]
+    ],
 }
 
-telco_response_example ={
+telco_response_example = {
     "startDate": "2023-09-20",
     "endDate": "2023-09-20",
     "results": [
@@ -138,50 +142,55 @@ def response_422():
             "endDate": "2024-05-16 20:41:48+00:00",
             "buildUrl": "https://ci-jenkins-xxx.com/job/your-tests/532",
             "jobStatus": "success",
-            "jobDuration": 3720
+            "jobDuration": 3720,
         },
-    ]
+    ],
 }
 
+
 def ocp_200_response():
     return response_200(ocp_response_example)
 
+
 def quay_200_response():
     return response_200(quay_response_example)
 
+
 def telco_200_response():
     return response_200(telco_response_example)
 
-cpt_response_example ={
-  "startDate": "2023-11-18",
-  "endDate": "2023-11-23",
-  "results": [
+
+cpt_response_example = {
+    "startDate": "2023-11-18",
+    "endDate": "2023-11-23",
+    "results": [
         {
-        "ciSystem": "PROW",
-        "uuid": "f6d084d5-b154-4108-b4f7-165094ccc838",
-        "releaseStream": "Nightly",
-        "jobStatus": "success",
-        "buildUrl": "https://ci..org/view/1726571333392797696",
-        "startDate": "2023-11-20T13:16:34Z",
-        "endDate": "2023-11-20T13:28:48Z",
-        "product": "ocp",
-        "version": "4.13",
-        "testName": "cluster-density-v2"
+            "ciSystem": "PROW",
+            "uuid": "f6d084d5-b154-4108-b4f7-165094ccc838",
+            "releaseStream": "Nightly",
+            "jobStatus": "success",
+            "buildUrl": "https://ci..org/view/1726571333392797696",
+            "startDate": "2023-11-20T13:16:34Z",
+            "endDate": "2023-11-20T13:28:48Z",
+            "product": "ocp",
+            "version": "4.13",
+            "testName": "cluster-density-v2",
         },
         {
-        "ciSystem": "JENKINS",
-        "uuid": "5b729011-3b4d-4ec4-953d-6881ac9da505",
-        "releaseStream": "Stable",
-        "jobStatus": "success",
-        "buildUrl": "https://ci..org/view/1726571333392797696",
-        "startDate": "2023-11-20T13:16:30Z",
-        "endDate": "2023-11-20T13:30:40Z",
-        "product": "ocp",
-        "version": "4.14",
-        "testName": "node-density-heavy"
+            "ciSystem": "JENKINS",
+            "uuid": "5b729011-3b4d-4ec4-953d-6881ac9da505",
+            "releaseStream": "Stable",
+            "jobStatus": "success",
+            "buildUrl": "https://ci..org/view/1726571333392797696",
+            "startDate": "2023-11-20T13:16:30Z",
+            "endDate": "2023-11-20T13:30:40Z",
+            "product": "ocp",
+            "version": "4.14",
+            "testName": "node-density-heavy",
         },
-    ]
+    ],
 }
 
+
 def cpt_200_response():
     return response_200(cpt_response_example)
diff --git a/backend/app/api/v1/commons/hasher.py b/backend/app/api/v1/commons/hasher.py
@@ -2,40 +2,42 @@
 import hashlib
 from cryptography.fernet import Fernet
 
-symmetric_encryptor = b'k3tGwuK6O59c0SEMmnIeJUEpTN5kuxibPy8Q8VfYC6A='
+symmetric_encryptor = b"k3tGwuK6O59c0SEMmnIeJUEpTN5kuxibPy8Q8VfYC6A="
+
 
 def hash_encrypt_json(json_data):
     # Serialize the JSON data to a string
     json_str = str(json_data)
-    
+
     # Generate an MD5 hash of the JSON string
     hash_digest = hashlib.md5(json_str.encode()).hexdigest()
-    
+
     # Compress the JSON string
     compressed_data = zlib.compress(json_str.encode())
-    
+
     cipher = Fernet(symmetric_encryptor)
-    
+
     # Encrypt the compressed JSON string
     encrypted_data = cipher.encrypt(compressed_data)
-    
+
     return hash_digest, encrypted_data
 
+
 def decrypt_unhash_json(hash_digest, encrypted_data):
     cipher = Fernet(symmetric_encryptor)
-    
+
     # Decrypt the encrypted JSON data
     decompressed_data = cipher.decrypt(encrypted_data)
-    
+
     # Decompress the decrypted data
     decompressed_json_str = zlib.decompress(decompressed_data).decode()
-    
+
     # Verify hash digest
     calculated_hash = hashlib.md5(decompressed_json_str.encode()).hexdigest()
     if calculated_hash != hash_digest:
         raise ValueError("Hash digest does not match")
-    
+
     # Deserialize the JSON string back to JSON data
     json_data = eval(decompressed_json_str)
-    
+
     return json_data
diff --git a/backend/app/api/v1/commons/hce.py b/backend/app/api/v1/commons/hce.py
@@ -5,26 +5,21 @@
 
 async def getData(start_datetime: date, end_datetime: date, configpath: str):
     query = {
-        "query": {
-            "bool": {
-                "filter": {
-                    "range": {
-                        "date": {
-                            "format": "yyyy-MM-dd"
-                        }
-                    }
-                }
-            }
-        }
+        "query": {"bool": {"filter": {"range": {"date": {"format": "yyyy-MM-dd"}}}}}
     }
 
     es = ElasticService(configpath=configpath)
-    response = await es.post(query=query, start_date=start_datetime, end_date=end_datetime, timestamp_field='date')
+    response = await es.post(
+        query=query,
+        start_date=start_datetime,
+        end_date=end_datetime,
+        timestamp_field="date",
+    )
     await es.close()
-    tasks = [item['_source'] for item in response]
+    tasks = [item["_source"] for item in response]
     jobs = pd.json_normalize(tasks)
-    jobs[['group']] = jobs[['group']].fillna(0)
-    jobs.fillna('', inplace=True)
+    jobs[["group"]] = jobs[["group"]].fillna(0)
+    jobs.fillna("", inplace=True)
     if len(jobs) == 0:
         return jobs
     return jobs
diff --git a/backend/app/api/v1/commons/ocm.py b/backend/app/api/v1/commons/ocm.py
@@ -7,36 +7,35 @@ async def getData(start_datetime: date, end_datetime: date, configpath: str):
     query = {
         "query": {
             "bool": {
-                "filter": {
-                    "range": {
-                        "metrics.earliest": {
-                            "format": "yyyy-MM-dd"
-                        }
-                    }
-                }
+                "filter": {"range": {"metrics.earliest": {"format": "yyyy-MM-dd"}}}
             }
         }
     }
 
     es = ElasticService(configpath=configpath)
-    response = await es.post(query=query, start_date=start_datetime, end_date=end_datetime, timestamp_field='metrics.earliest')
+    response = await es.post(
+        query=query,
+        start_date=start_datetime,
+        end_date=end_datetime,
+        timestamp_field="metrics.earliest",
+    )
     await es.close()
-    tasks = [item['_source'] for item in response]
+    tasks = [item["_source"] for item in response]
     jobs = pd.json_normalize(tasks)
     if len(jobs) == 0:
         return jobs
 
-    if 'buildUrl' not in jobs.columns:
+    if "buildUrl" not in jobs.columns:
         jobs.insert(len(jobs.columns), "buildUrl", "")
-    if 'ciSystem' not in jobs.columns:
+    if "ciSystem" not in jobs.columns:
         jobs.insert(len(jobs.columns), "ciSystem", "")
-    jobs.fillna('', inplace=True)
-    jobs['jobStatus'] = jobs.apply(convertJobStatus, axis=1)
+    jobs.fillna("", inplace=True)
+    jobs["jobStatus"] = jobs.apply(convertJobStatus, axis=1)
     return jobs
 
 
 def fillCiSystem(row):
-    currDate = datetime.strptime(row["metrics.earliest"][:26], '%Y-%m-%dT%H:%M:%S.%f')
+    currDate = datetime.strptime(row["metrics.earliest"][:26], "%Y-%m-%dT%H:%M:%S.%f")
     if currDate > datetime(2024, 6, 24):
         return "Jenkins"
     else: