Logic to use internal ES instance

README.md

@@ -41,8 +41,18 @@ indice=
 username=
 password=
 ```
+If you also have an archived internal instance that keeps track of older data, it can be specified with '.internal' suffix. Example of our `OCP` internal archived instance's configuration.
+```toml
+[ocp.elasticsearch.internal]
+url=
+indice=
+# [optional] common prefix for all indexes
+prefix=
+username=
+password=
+```
 
-Internally the API when serving the `/ocp` enpoints will use this connection.
+Internally the API when serving the `/ocp` enpoints will use this connection. Also it is suggested to create indexes with same name in the archived instances too to avoid further complications.
 
 The `jira` table requires a `url` key and a `personal_access_token` key. The `url` is a string value that points to the URL address of your Jira resource. The [Personal Access Token](https://confluence.atlassian.com/enterprise/using-personal-access-tokens-1026032365.html) is a string value that is the credential issued to authenticate and authorize this application with your Jira resource. 
 

backend/app/api/v1/commons/hce.py

@@ -17,13 +17,11 @@ async def getData(start_datetime: date, end_datetime: date, configpath: str):
             }
         }
     }
-    query['query']['bool']['filter']['range']['date']['lte'] = str(end_datetime)
-    query['query']['bool']['filter']['range']['date']['gte'] = str(start_datetime)
 
     es = ElasticService(configpath=configpath)
-    response = await es.post(query)
+    response = await es.post(query=query, start_date=start_datetime, end_date=end_datetime, timestamp_field='date')
     await es.close()
-    tasks = [item['_source'] for item in response["hits"]["hits"]]
+    tasks = [item['_source'] for item in response]
     jobs = pd.json_normalize(tasks)
     jobs[['group']] = jobs[['group']].fillna(0)
     jobs.fillna('', inplace=True)

backend/app/api/v1/commons/ocp.py

@@ -17,13 +17,11 @@ async def getData(start_datetime: date, end_datetime: date, configpath: str):
             }
         }
     }
-    query['query']['bool']['filter']['range']['timestamp']['lte'] = str(end_datetime)
-    query['query']['bool']['filter']['range']['timestamp']['gte'] = str(start_datetime)
 
     es = ElasticService(configpath=configpath)
-    response = await es.post(query)
+    response = await es.post(query=query, start_date=start_datetime, end_date=end_datetime, timestamp_field='timestamp')
     await es.close()
-    tasks = [item['_source'] for item in response["hits"]["hits"]]
+    tasks = [item['_source'] for item in response]
     jobs = pd.json_normalize(tasks)
     if len(jobs) == 0:
         return jobs

backend/app/api/v1/commons/utils.py

@@ -11,7 +11,7 @@ async def getMetadata(uuid: str, configpath: str) :
     }
     print(query)
     es = ElasticService(configpath=configpath)
-    response = await es.post(query)
+    response = await es.post(query=query)
     await es.close()
-    meta = [item['_source'] for item in response["hits"]["hits"]]
-    return meta[0]
+    meta = [item['_source'] for item in response]
+    return meta[0]

backend/app/api/v1/endpoints/ocp/graph.py

@@ -218,9 +218,9 @@ async def jobSummary(uuids: list):
     }
     print(query)
     es = ElasticService(configpath="ocp.elasticsearch",index=index)
-    response = await es.post(query)
+    response = await es.post(query=query)
     await es.close()
-    runs = [item['_source'] for item in response["hits"]["hits"]]
+    runs = [item['_source'] for item in response]
     return runs
 
 async def processBurner(data: dict) :
@@ -344,9 +344,9 @@ async def getBurnerResults(uuid: str, uuids: list, index: str ):
     }
     print(query)
     es = ElasticService(configpath="ocp.elasticsearch",index=index)
-    response = await es.post(query)
+    response = await es.post(query=query)
     await es.close()
-    runs = [item['_source'] for item in response["hits"]["hits"]]
+    runs = [item['_source'] for item in response]
     return runs
 
 async def getResults(uuid: str, uuids: list, index: str ):
@@ -364,9 +364,9 @@ async def getResults(uuid: str, uuids: list, index: str ):
     }
     print(query)
     es = ElasticService(configpath="ocp.elasticsearch",index=index)
-    response = await es.post(query)
+    response = await es.post(query=query)
     await es.close()
-    runs = [item['_source'] for item in response["hits"]["hits"]]
+    runs = [item['_source'] for item in response]
     return runs
 
 async def getMatchRuns(meta: dict, workerCount: False):
@@ -414,9 +414,9 @@ async def getMatchRuns(meta: dict, workerCount: False):
 
     print(query)
     es = ElasticService(configpath="ocp.elasticsearch")
-    response = await es.post(query)
+    response = await es.post(query=query)
     await es.close()
-    runs = [item['_source'] for item in response["hits"]["hits"]]
+    runs = [item['_source'] for item in response]
     uuids = []
     for run in runs :
         uuids.append(run["uuid"])

backend/app/api/v1/endpoints/ocp/results.py

@@ -21,7 +21,7 @@ async def results_for_job(
     }
 
     es = ElasticService(configpath="ocp.elasticsearch")
-    response = await es.post(query)
+    response = await es.post(query=query)
     await es.close()
-    tasks = [item['_source'] for item in response["hits"]["hits"]]
+    tasks = [item['_source'] for item in response]
     return tasks

backend/app/api/v1/endpoints/quay/quayGraphs.py

@@ -308,9 +308,9 @@ async def getMatchRuns(meta: dict):
 
     print(query)
     es = ElasticService(configpath="quay.elasticsearch")
-    response = await es.post(query)
+    response = await es.post(query=query)
     await es.close()
-    runs = [item['_source'] for item in response["hits"]["hits"]]
+    runs = [item['_source'] for item in response]
     uuids = []
     for run in runs :
         uuids.append(run["uuid"])