Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor the Python files for consistent formatting #139

Merged
merged 1 commit into from
Dec 5, 2024
Merged

Refactor the Python files for consistent formatting #139

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions backend/app/api/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,26 +16,26 @@
router = APIRouter()

# OCP endpoints
router.include_router(ocpJobs.router, tags=['ocp'])
router.include_router(results.router, tags=['ocp'])
router.include_router(graph.router, tags=['ocp.graphs'])
router.include_router(ocpJobs.router, tags=["ocp"])
router.include_router(results.router, tags=["ocp"])
router.include_router(graph.router, tags=["ocp.graphs"])

# CPT endpoints
router.include_router(cptJobs.router, tags=['cpt'])
router.include_router(cptJobs.router, tags=["cpt"])

# Quay endpoints
router.include_router(quayJobs.router, tags=['quay'])
router.include_router(quayGraphs.router, tags=['quay'])
router.include_router(quayJobs.router, tags=["quay"])
router.include_router(quayGraphs.router, tags=["quay"])

# Telco endpoints
router.include_router(telcoJobs.router, tags=['telco'])
router.include_router(telcoGraphs.router, tags=['telco'])
router.include_router(telcoJobs.router, tags=["telco"])
router.include_router(telcoGraphs.router, tags=["telco"])

# Jira endpoints
router.include_router(jira.router, tags=['jira'])
router.include_router(jira.router, tags=["jira"])

# Horreum endpoint
router.include_router(horreum.router, tags=['horreum'])
router.include_router(horreum.router, tags=["horreum"])

# OCM endpoint
router.include_router(ocmJobs.router, tags=['ocm'])
router.include_router(ocmJobs.router, tags=["ocm"])
97 changes: 53 additions & 44 deletions backend/app/api/v1/commons/example_responses.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,26 @@
def response_200(example):
return {
"content": {
"application/json": {
"example": example,
}
},
"content": {
"application/json": {
"example": example,
}
},
}


def response_422():
return {
"content": {
"application/json": {
"example": {"error": "invalid date format, start_date must be less than end_date"},
}
"content": {
"application/json": {
"example": {
"error": "invalid date format, start_date must be less than end_date"
},
}
},
}

ocp_response_example ={

ocp_response_example = {
"startDate": "2023-09-20",
"endDate": "2023-09-20",
"results": [
Expand Down Expand Up @@ -47,7 +51,7 @@ def response_422():
"startDate": "2023-09-20T02:14:07Z",
"endDate": "2023-09-20T03:41:48Z",
"timestamp": "2023-09-20T02:14:07Z",
"shortVersion": "4.14"
"shortVersion": "4.14",
},
{
"ciSystem": "PROW",
Expand Down Expand Up @@ -76,12 +80,12 @@ def response_422():
"startDate": "2023-09-20T07:19:00Z",
"endDate": "2023-09-20T07:28:42Z",
"timestamp": "2023-09-20T07:19:00Z",
"shortVersion": "4.13"
"shortVersion": "4.13",
},
]
],
}

quay_response_example ={
quay_response_example = {
"startDate": "2023-09-20",
"endDate": "2023-09-20",
"results": [
Expand Down Expand Up @@ -114,12 +118,12 @@ def response_422():
"startDate": "2023-09-20T02:14:07Z",
"endDate": "2023-09-20T03:41:48Z",
"timestamp": "2023-09-20T02:14:07Z",
"shortVersion": "4.14"
"shortVersion": "4.14",
},
]
],
}

telco_response_example ={
telco_response_example = {
"startDate": "2023-09-20",
"endDate": "2023-09-20",
"results": [
Expand All @@ -138,50 +142,55 @@ def response_422():
"endDate": "2024-05-16 20:41:48+00:00",
"buildUrl": "https://ci-jenkins-xxx.com/job/your-tests/532",
"jobStatus": "success",
"jobDuration": 3720
"jobDuration": 3720,
},
]
],
}


def ocp_200_response():
return response_200(ocp_response_example)


def quay_200_response():
return response_200(quay_response_example)


def telco_200_response():
return response_200(telco_response_example)

cpt_response_example ={
"startDate": "2023-11-18",
"endDate": "2023-11-23",
"results": [

cpt_response_example = {
"startDate": "2023-11-18",
"endDate": "2023-11-23",
"results": [
{
"ciSystem": "PROW",
"uuid": "f6d084d5-b154-4108-b4f7-165094ccc838",
"releaseStream": "Nightly",
"jobStatus": "success",
"buildUrl": "https://ci..org/view/1726571333392797696",
"startDate": "2023-11-20T13:16:34Z",
"endDate": "2023-11-20T13:28:48Z",
"product": "ocp",
"version": "4.13",
"testName": "cluster-density-v2"
"ciSystem": "PROW",
"uuid": "f6d084d5-b154-4108-b4f7-165094ccc838",
"releaseStream": "Nightly",
"jobStatus": "success",
"buildUrl": "https://ci..org/view/1726571333392797696",
"startDate": "2023-11-20T13:16:34Z",
"endDate": "2023-11-20T13:28:48Z",
"product": "ocp",
"version": "4.13",
"testName": "cluster-density-v2",
},
{
"ciSystem": "JENKINS",
"uuid": "5b729011-3b4d-4ec4-953d-6881ac9da505",
"releaseStream": "Stable",
"jobStatus": "success",
"buildUrl": "https://ci..org/view/1726571333392797696",
"startDate": "2023-11-20T13:16:30Z",
"endDate": "2023-11-20T13:30:40Z",
"product": "ocp",
"version": "4.14",
"testName": "node-density-heavy"
"ciSystem": "JENKINS",
"uuid": "5b729011-3b4d-4ec4-953d-6881ac9da505",
"releaseStream": "Stable",
"jobStatus": "success",
"buildUrl": "https://ci..org/view/1726571333392797696",
"startDate": "2023-11-20T13:16:30Z",
"endDate": "2023-11-20T13:30:40Z",
"product": "ocp",
"version": "4.14",
"testName": "node-density-heavy",
},
]
],
}


def cpt_200_response():
return response_200(cpt_response_example)
24 changes: 13 additions & 11 deletions backend/app/api/v1/commons/hasher.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,40 +2,42 @@
import hashlib
from cryptography.fernet import Fernet

symmetric_encryptor = b'k3tGwuK6O59c0SEMmnIeJUEpTN5kuxibPy8Q8VfYC6A='
symmetric_encryptor = b"k3tGwuK6O59c0SEMmnIeJUEpTN5kuxibPy8Q8VfYC6A="


def hash_encrypt_json(json_data):
# Serialize the JSON data to a string
json_str = str(json_data)

# Generate an MD5 hash of the JSON string
hash_digest = hashlib.md5(json_str.encode()).hexdigest()

# Compress the JSON string
compressed_data = zlib.compress(json_str.encode())

cipher = Fernet(symmetric_encryptor)

# Encrypt the compressed JSON string
encrypted_data = cipher.encrypt(compressed_data)

return hash_digest, encrypted_data


def decrypt_unhash_json(hash_digest, encrypted_data):
cipher = Fernet(symmetric_encryptor)

# Decrypt the encrypted JSON data
decompressed_data = cipher.decrypt(encrypted_data)

# Decompress the decrypted data
decompressed_json_str = zlib.decompress(decompressed_data).decode()

# Verify hash digest
calculated_hash = hashlib.md5(decompressed_json_str.encode()).hexdigest()
if calculated_hash != hash_digest:
raise ValueError("Hash digest does not match")

# Deserialize the JSON string back to JSON data
json_data = eval(decompressed_json_str)

return json_data
25 changes: 10 additions & 15 deletions backend/app/api/v1/commons/hce.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,21 @@

async def getData(start_datetime: date, end_datetime: date, configpath: str):
query = {
"query": {
"bool": {
"filter": {
"range": {
"date": {
"format": "yyyy-MM-dd"
}
}
}
}
}
"query": {"bool": {"filter": {"range": {"date": {"format": "yyyy-MM-dd"}}}}}
}

es = ElasticService(configpath=configpath)
response = await es.post(query=query, start_date=start_datetime, end_date=end_datetime, timestamp_field='date')
response = await es.post(
query=query,
start_date=start_datetime,
end_date=end_datetime,
timestamp_field="date",
)
await es.close()
tasks = [item['_source'] for item in response]
tasks = [item["_source"] for item in response]
jobs = pd.json_normalize(tasks)
jobs[['group']] = jobs[['group']].fillna(0)
jobs.fillna('', inplace=True)
jobs[["group"]] = jobs[["group"]].fillna(0)
jobs.fillna("", inplace=True)
if len(jobs) == 0:
return jobs
return jobs
27 changes: 13 additions & 14 deletions backend/app/api/v1/commons/ocm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,36 +7,35 @@ async def getData(start_datetime: date, end_datetime: date, configpath: str):
query = {
"query": {
"bool": {
"filter": {
"range": {
"metrics.earliest": {
"format": "yyyy-MM-dd"
}
}
}
"filter": {"range": {"metrics.earliest": {"format": "yyyy-MM-dd"}}}
}
}
}

es = ElasticService(configpath=configpath)
response = await es.post(query=query, start_date=start_datetime, end_date=end_datetime, timestamp_field='metrics.earliest')
response = await es.post(
query=query,
start_date=start_datetime,
end_date=end_datetime,
timestamp_field="metrics.earliest",
)
await es.close()
tasks = [item['_source'] for item in response]
tasks = [item["_source"] for item in response]
jobs = pd.json_normalize(tasks)
if len(jobs) == 0:
return jobs

if 'buildUrl' not in jobs.columns:
if "buildUrl" not in jobs.columns:
jobs.insert(len(jobs.columns), "buildUrl", "")
if 'ciSystem' not in jobs.columns:
if "ciSystem" not in jobs.columns:
jobs.insert(len(jobs.columns), "ciSystem", "")
jobs.fillna('', inplace=True)
jobs['jobStatus'] = jobs.apply(convertJobStatus, axis=1)
jobs.fillna("", inplace=True)
jobs["jobStatus"] = jobs.apply(convertJobStatus, axis=1)
return jobs


def fillCiSystem(row):
currDate = datetime.strptime(row["metrics.earliest"][:26], '%Y-%m-%dT%H:%M:%S.%f')
currDate = datetime.strptime(row["metrics.earliest"][:26], "%Y-%m-%dT%H:%M:%S.%f")
if currDate > datetime(2024, 6, 24):
return "Jenkins"
else:
Expand Down
Loading