From da54fb8fe1ac1681ab36f84b5f885054ba43f0cd Mon Sep 17 00:00:00 2001 From: MVarshini Date: Fri, 25 Oct 2024 23:28:11 +0530 Subject: [PATCH 1/7] Page Size and offset --- backend/app/api/v1/endpoints/ocp/ocpJobs.py | 2 +- backend/app/services/search.py | 2 +- frontend/src/components/templates/Home/index.jsx | 4 ++-- frontend/src/reducers/homeReducer.js | 2 -- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/backend/app/api/v1/endpoints/ocp/ocpJobs.py b/backend/app/api/v1/endpoints/ocp/ocpJobs.py index 5637b29d..aeddc151 100644 --- a/backend/app/api/v1/endpoints/ocp/ocpJobs.py +++ b/backend/app/api/v1/endpoints/ocp/ocpJobs.py @@ -12,7 +12,7 @@ @router.get( "/api/v1/ocp/jobs", summary="Returns a job list", - description="Returns a list of jobs in the specified dates. \ + description="Returns a list of jobs in the specified dates of requested size. \ If not dates are provided the API will default the values. \ `startDate`: will be set to the day of the request minus 5 days.\ `endDate`: will be set to the day of the request.", diff --git a/backend/app/services/search.py b/backend/app/services/search.py index 9f1837fb..f9c9e53d 100644 --- a/backend/app/services/search.py +++ b/backend/app/services/search.py @@ -47,7 +47,7 @@ async def post( self, query, indice=None, - size=10000, + size=None, start_date=None, end_date=None, timestamp_field=None, diff --git a/frontend/src/components/templates/Home/index.jsx b/frontend/src/components/templates/Home/index.jsx index 52b281b0..d02d0ee5 100644 --- a/frontend/src/components/templates/Home/index.jsx +++ b/frontend/src/components/templates/Home/index.jsx @@ -22,7 +22,7 @@ const Home = () => { tableColumns, activeSortDir, activeSortIndex, - tableData, + results, filterOptions, tableFilters, categoryFilterValue, @@ -91,7 +91,7 @@ const Home = () => { /> { return { ...state, activeSortIndex: payload }; case TYPES.SET_CPT_SORT_DIR: return { ...state, activeSortDir: payload }; - case TYPES.SET_CPT_INIT_JOBS: - return { ...state, tableData: payload }; case TYPES.SET_CPT_FILTER_DATA: return { ...state, filterData: payload }; case TYPES.SET_CATEGORY_FILTER: From 5907b40e8210c5b7a3504a373699797f7417a1f8 Mon Sep 17 00:00:00 2001 From: MVarshini Date: Sat, 9 Nov 2024 17:19:02 +0530 Subject: [PATCH 2/7] pagination params --- backend/app/api/v1/commons/hce.py | 5 +++-- backend/app/api/v1/endpoints/cpt/maps/hce.py | 3 +-- frontend/src/components/templates/Home/index.jsx | 4 ++-- frontend/src/reducers/homeReducer.js | 2 ++ 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/app/api/v1/commons/hce.py b/backend/app/api/v1/commons/hce.py index a9b66e7a..f397ce0a 100644 --- a/backend/app/api/v1/commons/hce.py +++ b/backend/app/api/v1/commons/hce.py @@ -7,7 +7,9 @@ async def getData( start_datetime: date, end_datetime: date, size: int, offset: int, configpath: str ): query = { - "query": {"bool": {"filter": {"range": {"date": {"format": "yyyy-MM-dd"}}}}} + "size": size, + "from": offset, + "query": {"bool": {"filter": {"range": {"date": {"format": "yyyy-MM-dd"}}}}}, } es = ElasticService(configpath=configpath) response = await es.post( @@ -25,5 +27,4 @@ async def getData( jobs[["group"]] = jobs[["group"]].fillna(0) jobs.fillna("", inplace=True) - return {"data": jobs, "total": response["total"]} diff --git a/backend/app/api/v1/endpoints/cpt/maps/hce.py b/backend/app/api/v1/endpoints/cpt/maps/hce.py index 3c812d0d..e7439b9a 100644 --- a/backend/app/api/v1/endpoints/cpt/maps/hce.py +++ b/backend/app/api/v1/endpoints/cpt/maps/hce.py @@ -2,6 +2,7 @@ from datetime import date import pandas as pd + ################################################################ # This will return a Dictionary from HCE required by the CPT # endpoint, it contians totalJobs and a Dataframe with the following columns: @@ -16,8 +17,6 @@ # "version" # "testName" ################################################################ - - async def hceMapper(start_datetime: date, end_datetime: date, size: int, offset: int): response = await getData( start_datetime, end_datetime, size, offset, f"hce.elasticsearch" diff --git a/frontend/src/components/templates/Home/index.jsx b/frontend/src/components/templates/Home/index.jsx index d02d0ee5..52b281b0 100644 --- a/frontend/src/components/templates/Home/index.jsx +++ b/frontend/src/components/templates/Home/index.jsx @@ -22,7 +22,7 @@ const Home = () => { tableColumns, activeSortDir, activeSortIndex, - results, + tableData, filterOptions, tableFilters, categoryFilterValue, @@ -91,7 +91,7 @@ const Home = () => { /> { return { ...state, activeSortIndex: payload }; case TYPES.SET_CPT_SORT_DIR: return { ...state, activeSortDir: payload }; + case TYPES.SET_CPT_INIT_JOBS: + return { ...state, tableData: payload }; case TYPES.SET_CPT_FILTER_DATA: return { ...state, filterData: payload }; case TYPES.SET_CATEGORY_FILTER: From 751f376ad22eb267ee53406e3714ad41f53bb2bd Mon Sep 17 00:00:00 2001 From: MVarshini Date: Fri, 29 Nov 2024 11:07:27 +0530 Subject: [PATCH 3/7] Sorting --- backend/app/api/v1/commons/ocp.py | 10 ++- backend/app/api/v1/commons/quay.py | 6 +- backend/app/api/v1/commons/telco.py | 2 + backend/app/api/v1/endpoints/ocp/ocpJobs.py | 40 +++++++++++ backend/app/api/v1/endpoints/quay/quayJobs.py | 40 +++++++++++ backend/app/services/splunk.py | 2 + frontend/src/actions/commonActions.js | 67 +---------------- frontend/src/actions/homeActions.js | 2 - frontend/src/actions/ocpActions.js | 2 - frontend/src/actions/sortingActions.js | 71 +++++++++++++++++-- frontend/src/actions/types.js | 2 + .../organisms/TableLayout/index.jsx | 11 ++- .../src/components/templates/Home/index.jsx | 1 + .../src/components/templates/OCP/index.jsx | 1 + .../src/components/templates/Quay/index.jsx | 1 + .../src/components/templates/Telco/index.jsx | 1 + frontend/src/reducers/homeReducer.js | 1 + frontend/src/reducers/ocpReducer.js | 3 + frontend/src/reducers/quayReducer.js | 3 + frontend/src/reducers/telcoReducer.js | 1 + 20 files changed, 188 insertions(+), 79 deletions(-) diff --git a/backend/app/api/v1/commons/ocp.py b/backend/app/api/v1/commons/ocp.py index 1eaa3c54..db0ed658 100644 --- a/backend/app/api/v1/commons/ocp.py +++ b/backend/app/api/v1/commons/ocp.py @@ -5,7 +5,12 @@ async def getData( - start_datetime: date, end_datetime: date, size: int, offset: int, configpath: str + start_datetime: date, + end_datetime: date, + size: int, + offset: int, + sort: str, + configpath: str, ): query = { "size": size, @@ -14,6 +19,9 @@ async def getData( "bool": {"filter": {"range": {"timestamp": {"format": "yyyy-MM-dd"}}}} }, } + if sort: + key, direction = sort.split(":") + query["sort"] = [{key: {"order": direction}}] es = ElasticService(configpath=configpath) response = await es.post( diff --git a/backend/app/api/v1/commons/quay.py b/backend/app/api/v1/commons/quay.py index 71deeb8f..b10874d4 100644 --- a/backend/app/api/v1/commons/quay.py +++ b/backend/app/api/v1/commons/quay.py @@ -5,7 +5,7 @@ async def getData( - start_datetime: date, end_datetime: date, size, offset, configpath: str + start_datetime: date, end_datetime: date, size, offset, sort: str, configpath: str ): query = { "size": size, @@ -15,6 +15,10 @@ async def getData( }, } + if sort: + key, direction = sort.split(":") + query["sort"] = [{key: {"order": direction}}] + es = ElasticService(configpath=configpath) response = await es.post( query=query, diff --git a/backend/app/api/v1/commons/telco.py b/backend/app/api/v1/commons/telco.py index b55e746f..91cba718 100644 --- a/backend/app/api/v1/commons/telco.py +++ b/backend/app/api/v1/commons/telco.py @@ -38,6 +38,8 @@ async def getData( "earliest_time": "{}T00:00:00".format(start_datetime.strftime("%Y-%m-%d")), "latest_time": "{}T23:59:59".format(end_datetime.strftime("%Y-%m-%d")), "output_mode": "json", + "sort_dir": "asc", + "sort_key": "test_type", } searchList = " OR ".join( ['test_type="{}"'.format(test_type) for test_type in test_types] diff --git a/backend/app/api/v1/endpoints/ocp/ocpJobs.py b/backend/app/api/v1/endpoints/ocp/ocpJobs.py index aeddc151..f161c50f 100644 --- a/backend/app/api/v1/endpoints/ocp/ocpJobs.py +++ b/backend/app/api/v1/endpoints/ocp/ocpJobs.py @@ -32,9 +32,16 @@ async def jobs( description="End date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-15"], ), +<<<<<<< HEAD pretty: bool = Query(False, description="Output content in pretty format."), size: int = Query(None, description="Number of jobs to fetch"), offset: int = Query(None, description="Offset Number to fetch jobs from"), +======= + pretty: bool = Query(False, description="Output contet in pretty format."), + size: int = Query(None, description="Number of jobs to fetch"), + offset: int = Query(None, description="Offset Number to fetch jobs from"), + sort: str = Query(None, descption="To sort fields on specified direction"), +>>>>>>> bd597e7 (Sorting) ): if start_date is None: start_date = datetime.utcnow().date() @@ -51,6 +58,7 @@ async def jobs( status_code=422, ) +<<<<<<< HEAD if offset and not size: raise HTTPException(400, f"offset {offset} specified without size") elif not offset and not size: @@ -71,6 +79,38 @@ async def jobs( "total": results["total"], "offset": offset + size, } +======= + if not offset: + offset = 0 + + if not size: + size = 10000 + offset = 0 + + if not sort: + sort = None + + results = await getData( + start_date, end_date, size, offset, sort, "ocp.elasticsearch" + ) + + if "data" in results and len(results["data"]) >= 1: + response = { + "startDate": start_date.__str__(), + "endDate": end_date.__str__(), + "results": results["data"].to_dict("records"), + "total": results["total"], + "offset": offset + size, + } + else: + response = { + "startDate": start_date.__str__(), + "endDate": end_date.__str__(), + "results": [], + "total": 0, + "offset": 0, + } +>>>>>>> bd597e7 (Sorting) if pretty: json_str = json.dumps(response, indent=4) diff --git a/backend/app/api/v1/endpoints/quay/quayJobs.py b/backend/app/api/v1/endpoints/quay/quayJobs.py index be05d074..3a212627 100644 --- a/backend/app/api/v1/endpoints/quay/quayJobs.py +++ b/backend/app/api/v1/endpoints/quay/quayJobs.py @@ -32,9 +32,16 @@ async def jobs( description="End date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-15"], ), +<<<<<<< HEAD pretty: bool = Query(False, description="Output content in pretty format."), size: int = Query(None, description="Number of jobs to fetch"), offset: int = Query(None, description="Offset Number to fetch jobs from"), +======= + pretty: bool = Query(False, description="Output contet in pretty format."), + size: int = Query(None, description="Number of jobs to fetch"), + offset: int = Query(None, description="Offset Number to fetch jobs from"), + sort: str = Query(None, descption="To sort fields on specified direction"), +>>>>>>> bd597e7 (Sorting) ): if start_date is None: start_date = datetime.utcnow().date() @@ -51,6 +58,7 @@ async def jobs( status_code=422, ) +<<<<<<< HEAD if offset and not size: raise HTTPException(400, f"offset {offset} specified without size") elif not offset and not size: @@ -72,6 +80,38 @@ async def jobs( "total": 0, "offset": 0, } +======= + if not offset: + offset = 0 + + if not size: + size = 10000 + offset = 0 + + if not sort: + sort = None + + results = await getData( + start_date, end_date, size, offset, sort, "quay.elasticsearch" + ) + + if "data" in results and len(results["data"]) >= 1: + response = { + "startDate": start_date.__str__(), + "endDate": end_date.__str__(), + "results": results["data"].to_dict("records"), + "total": results["total"], + "offset": offset + size, + } + else: + response = { + "startDate": start_date.__str__(), + "endDate": end_date.__str__(), + "results": [], + "total": 0, + "offset": 0, + } +>>>>>>> bd597e7 (Sorting) if pretty: json_str = json.dumps(response, indent=4) diff --git a/backend/app/services/splunk.py b/backend/app/services/splunk.py index 70b00263..d7cd8535 100644 --- a/backend/app/services/splunk.py +++ b/backend/app/services/splunk.py @@ -45,6 +45,8 @@ async def query( """ query["count"] = size query["offset"] = offset + query["sort_dir"] = "asc" + query["sort_key"] = "test_type" # If additional search parameters are provided, include those in searchindex searchindex = ( diff --git a/frontend/src/actions/commonActions.js b/frontend/src/actions/commonActions.js index 0551bd0f..2698a52c 100644 --- a/frontend/src/actions/commonActions.js +++ b/frontend/src/actions/commonActions.js @@ -6,69 +6,6 @@ import { setOCPCatFilters } from "./ocpActions"; import { setQuayCatFilters } from "./quayActions"; import { setTelcoCatFilters } from "./telcoActions"; -const getSortableRowValues = (result, tableColumns) => { - const tableKeys = tableColumns.map((item) => item.value); - return tableKeys.map((key) => result[key]); -}; - -export const sortTable = (currState) => (dispatch, getState) => { - const results = [...getState()[currState].filteredResults]; - const { activeSortDir, activeSortIndex, tableColumns } = - getState()[currState]; - try { - if (activeSortIndex !== null && typeof activeSortIndex !== "undefined") { - const sortedResults = results.sort((a, b) => { - const aValue = getSortableRowValues(a, tableColumns)[activeSortIndex]; - const bValue = getSortableRowValues(b, tableColumns)[activeSortIndex]; - if (typeof aValue === "number") { - if (activeSortDir === "asc") { - return aValue - bValue; - } - return bValue - aValue; - } else { - if (activeSortDir === "asc") { - return aValue.localeCompare(bValue); - } - return bValue.localeCompare(aValue); - } - }); - dispatch(sortedTableRows(currState, sortedResults)); - } - } catch (error) { - console.log(error); - } -}; - -const sortedTableRows = (currState, sortedResults) => (dispatch) => { - if (currState === "cpt") { - dispatch({ - type: TYPES.SET_FILTERED_DATA, - payload: sortedResults, - }); - return; - } - if (currState === "ocp") { - dispatch({ - type: TYPES.SET_OCP_FILTERED_DATA, - payload: sortedResults, - }); - return; - } - if (currState === "quay") { - dispatch({ - type: TYPES.SET_QUAY_FILTERED_DATA, - payload: sortedResults, - }); - return; - } - if (currState === "telco") { - dispatch({ - type: TYPES.SET_TELCO_FILTERED_DATA, - payload: sortedResults, - }); - } -}; - const findItemCount = (data, key, value) => { return data.reduce(function (n, item) { return n + (item[key].toLowerCase() === value); @@ -203,13 +140,15 @@ export const getSelectedFilter = }; export const getRequestParams = (type) => (dispatch, getState) => { - const { start_date, end_date, size, offset } = getState()[type]; + const { start_date, end_date, size, offset, sort } = getState()[type]; + // const sortParam = `${activeSortIndex}:${activeSortDir}`; const params = { pretty: true, ...(start_date && { start_date }), ...(end_date && { end_date }), size: size, offset: offset, + ...(sort && { sort }), }; return params; diff --git a/frontend/src/actions/homeActions.js b/frontend/src/actions/homeActions.js index 7a6dd735..fda6284b 100644 --- a/frontend/src/actions/homeActions.js +++ b/frontend/src/actions/homeActions.js @@ -9,7 +9,6 @@ import { getFilteredData, getRequestParams, getSelectedFilter, - sortTable, } from "./commonActions"; import API from "@/utils/axiosInstance"; @@ -56,7 +55,6 @@ export const fetchOCPJobsData = }); dispatch(applyFilters()); - dispatch(sortTable("cpt")); dispatch(tableReCalcValues()); } } catch (error) { diff --git a/frontend/src/actions/ocpActions.js b/frontend/src/actions/ocpActions.js index 94657b15..c7634c84 100644 --- a/frontend/src/actions/ocpActions.js +++ b/frontend/src/actions/ocpActions.js @@ -9,7 +9,6 @@ import { getFilteredData, getRequestParams, getSelectedFilter, - sortTable, } from "./commonActions"; import API from "@/utils/axiosInstance"; @@ -52,7 +51,6 @@ export const fetchOCPJobs = () => async (dispatch) => { }); dispatch(applyFilters()); - dispatch(sortTable("ocp")); dispatch(tableReCalcValues()); } } catch (error) { diff --git a/frontend/src/actions/sortingActions.js b/frontend/src/actions/sortingActions.js index 742820b5..15e72cb8 100644 --- a/frontend/src/actions/sortingActions.js +++ b/frontend/src/actions/sortingActions.js @@ -1,9 +1,18 @@ +import * as TYPES from "@/actions/types.js"; + +import { fetchOCPJobs, setOCPSortDir, setOCPSortIndex } from "./ocpActions"; +import { + fetchQuayJobsData, + setQuaySortDir, + setQuaySortIndex, +} from "./quayActions"; +import { + fetchTelcoJobsData, + setTelcoSortDir, + setTelcoSortIndex, +} from "./telcoActions"; import { setCPTSortDir, setCPTSortIndex } from "./homeActions"; -import { setOCPSortDir, setOCPSortIndex } from "./ocpActions"; -import { setQuaySortDir, setQuaySortIndex } from "./quayActions"; -import { setTelcoSortDir, setTelcoSortIndex } from "./telcoActions"; -import { sortTable } from "./commonActions"; import store from "@/store/store"; const { dispatch } = store; @@ -29,6 +38,56 @@ export const setActiveSortIndex = (index, currType) => { dispatch(setTelcoSortIndex(index)); } }; -export const handleOnSort = (currType) => { - dispatch(sortTable(currType)); +export const handleOnSort = (colName, currType) => { + dispatch(sortTable(colName, currType)); +}; + +const offsetActions = { + cpt: TYPES.SET_CPT_OFFSET, + ocp: TYPES.SET_OCP_OFFSET, + quay: TYPES.SET_QUAY_OFFSET, + telco: TYPES.SET_TELCO_OFFSET, +}; +const fetchJobsMap = { + ocp: fetchOCPJobs, + quay: fetchQuayJobsData, + telco: fetchTelcoJobsData, +}; +const sortObjActions = { + ocp: TYPES.SET_OCP_SORT_OBJ, + quay: TYPES.SET_QUAY_SORT_OBJ, +}; +export const sortTable = (colName, currState) => (dispatch, getState) => { + const { activeSortDir, activeSortIndex } = getState()[currState]; + const countObj = [ + "masterNodesCount", + "workerNodesCount", + "infraNodesCount", + "totalNodesCount", + "startDate", + "endDate", + ]; + try { + if ( + typeof activeSortDir !== "undefined" && + typeof activeSortIndex !== "undefined" + ) { + dispatch({ type: offsetActions[currState], payload: 0 }); + let fieldName = countObj.includes(colName) + ? colName + : `${colName}.keyword`; + if (colName === "build") { + fieldName = "ocpVersion.keyword"; + } + + const sortParam = `${fieldName}:${activeSortDir}`; + dispatch({ type: sortObjActions[currState], payload: sortParam }); + console.log(sortParam); + const isFromSorting = true; + + dispatch(fetchJobsMap[currState](isFromSorting)); + } + } catch (error) { + console.log(error); + } }; diff --git a/frontend/src/actions/types.js b/frontend/src/actions/types.js index 734d6568..2677f164 100644 --- a/frontend/src/actions/types.js +++ b/frontend/src/actions/types.js @@ -33,6 +33,7 @@ export const SET_OCP_JOBS_DATA = "SET_OCP_JOBS_DATA"; export const SET_OCP_DATE_FILTER = "SET_OCP_DATE_FILTER"; export const SET_OCP_SORT_INDEX = "SET_OCP_SORT_INDEX"; export const SET_OCP_SORT_DIR = "SET_OCP_SORT_DIR"; +export const SET_OCP_SORT_OBJ = "SET_OCP_SORT_OBJ"; export const SET_OCP_PAGE = "SET_OCP_PAGE"; export const SET_OCP_PAGE_OPTIONS = "SET_OCP_PAGE_OPTIONS"; export const SET_OCP_INIT_JOBS = "SET_OCP_INIT_JOBS"; @@ -52,6 +53,7 @@ export const SET_QUAY_JOBS_DATA = "SET_QUAY_JOBS_DATA"; export const SET_QUAY_DATE_FILTER = "SET_QUAY_DATE_FILTER"; export const SET_QUAY_SORT_INDEX = "SET_QUAY_SORT_INDEX"; export const SET_QUAY_SORT_DIR = "SET_QUAY_SORT_DIR"; +export const SET_QUAY_SORT_OBJ = "SET_QUAY_SORT_OBJ"; export const SET_QUAY_PAGE = "SET_QUAY_PAGE"; export const SET_QUAY_PAGE_OPTIONS = "SET_QUAY_PAGE_OPTIONS"; export const SET_QUAY_INIT_JOBS = "SET_QUAY_INIT_JOBS"; diff --git a/frontend/src/components/organisms/TableLayout/index.jsx b/frontend/src/components/organisms/TableLayout/index.jsx index 500c1bc9..e24def32 100644 --- a/frontend/src/components/organisms/TableLayout/index.jsx +++ b/frontend/src/components/organisms/TableLayout/index.jsx @@ -21,9 +21,10 @@ const TableLayout = (props) => { totalItems, addExpansion, type, + shouldSort, } = props; - const getSortParams = (columnIndex) => ({ + const getSortParams = (columnIndex, colName) => ({ sortBy: { index: activeSortIndex, direction: activeSortDir, @@ -32,7 +33,7 @@ const TableLayout = (props) => { onSort: (_event, index, direction) => { setActiveSortIndex(index, type); setActiveSortDir(direction, type); - handleOnSort(type); + handleOnSort(colName, type); }, columnIndex, }); @@ -46,7 +47,10 @@ const TableLayout = (props) => { {tableColumns?.length > 0 && tableColumns.map((col, idx) => ( - + {col.name} ))} @@ -99,5 +103,6 @@ TableLayout.propTypes = { type: PropTypes.string, isRunExpanded: PropTypes.func, setRunExpanded: PropTypes.func, + shouldSort: PropTypes.bool, }; export default TableLayout; diff --git a/frontend/src/components/templates/Home/index.jsx b/frontend/src/components/templates/Home/index.jsx index 52b281b0..f18264b4 100644 --- a/frontend/src/components/templates/Home/index.jsx +++ b/frontend/src/components/templates/Home/index.jsx @@ -100,6 +100,7 @@ const Home = () => { totalItems={totalJobs} addExpansion={false} type={"cpt"} + shouldSort={false} /> ); diff --git a/frontend/src/components/templates/OCP/index.jsx b/frontend/src/components/templates/OCP/index.jsx index 74db5d00..1e21057f 100644 --- a/frontend/src/components/templates/OCP/index.jsx +++ b/frontend/src/components/templates/OCP/index.jsx @@ -135,6 +135,7 @@ const OCP = () => { setRunExpanded={setRunExpanded} graphData={graphData} type={"ocp"} + shouldSort={true} /> ); diff --git a/frontend/src/components/templates/Quay/index.jsx b/frontend/src/components/templates/Quay/index.jsx index 23e0ed40..bff5c2a4 100644 --- a/frontend/src/components/templates/Quay/index.jsx +++ b/frontend/src/components/templates/Quay/index.jsx @@ -134,6 +134,7 @@ const Quay = () => { isRunExpanded={isRunExpanded} setRunExpanded={setRunExpanded} graphData={graphData} + shouldSort={true} /> ); diff --git a/frontend/src/components/templates/Telco/index.jsx b/frontend/src/components/templates/Telco/index.jsx index 181b9923..110cf42d 100644 --- a/frontend/src/components/templates/Telco/index.jsx +++ b/frontend/src/components/templates/Telco/index.jsx @@ -133,6 +133,7 @@ const Telco = () => { isRunExpanded={isRunExpanded} setRunExpanded={setRunExpanded} graphData={graphData} + shouldSort={false} /> ); diff --git a/frontend/src/reducers/homeReducer.js b/frontend/src/reducers/homeReducer.js index a2f477e1..716c4d47 100644 --- a/frontend/src/reducers/homeReducer.js +++ b/frontend/src/reducers/homeReducer.js @@ -32,6 +32,7 @@ const initialState = { filterData: [], activeSortDir: null, activeSortIndex: null, + sort: "", categoryFilterValue: "", filterOptions: [], appliedFilters: {}, diff --git a/frontend/src/reducers/ocpReducer.js b/frontend/src/reducers/ocpReducer.js index 20232ad8..78f1df2e 100644 --- a/frontend/src/reducers/ocpReducer.js +++ b/frontend/src/reducers/ocpReducer.js @@ -42,6 +42,7 @@ const initialState = { ], activeSortDir: null, activeSortIndex: null, + sort: "", page: START_PAGE, perPage: DEFAULT_PER_PAGE, size: DEFAULT_PER_PAGE, @@ -133,6 +134,8 @@ const OCPReducer = (state = initialState, action = {}) => { return { ...state, activeSortIndex: payload }; case TYPES.SET_OCP_SORT_DIR: return { ...state, activeSortDir: payload }; + case TYPES.SET_OCP_SORT_OBJ: + return { ...state, sort: payload }; case TYPES.SET_OCP_PAGE: return { ...state, page: payload }; case TYPES.SET_OCP_PAGE_OPTIONS: diff --git a/frontend/src/reducers/quayReducer.js b/frontend/src/reducers/quayReducer.js index 9e33d160..db7f4551 100644 --- a/frontend/src/reducers/quayReducer.js +++ b/frontend/src/reducers/quayReducer.js @@ -61,6 +61,7 @@ const initialState = { appliedFilters: {}, activeSortDir: null, activeSortIndex: null, + sort: "", graphData: [], page: START_PAGE, perPage: DEFAULT_PER_PAGE, @@ -97,6 +98,8 @@ const QuayReducer = (state = initialState, action = {}) => { return { ...state, activeSortIndex: payload }; case TYPES.SET_QUAY_SORT_DIR: return { ...state, activeSortDir: payload }; + case TYPES.SET_QUAY_SORT_OBJ: + return { ...state, sort: payload }; case TYPES.SET_QUAY_PAGE: return { ...state, page: payload }; case TYPES.SET_QUAY_PAGE_OPTIONS: diff --git a/frontend/src/reducers/telcoReducer.js b/frontend/src/reducers/telcoReducer.js index 5dd1a140..1637f131 100644 --- a/frontend/src/reducers/telcoReducer.js +++ b/frontend/src/reducers/telcoReducer.js @@ -58,6 +58,7 @@ const initialState = { appliedFilters: {}, activeSortDir: null, activeSortIndex: null, + sort: "", graphData: [], page: START_PAGE, perPage: DEFAULT_PER_PAGE, From 49f65459ef13e11589b2eaa0354364eb2f7d1a92 Mon Sep 17 00:00:00 2001 From: MVarshini Date: Fri, 13 Dec 2024 18:56:45 +0530 Subject: [PATCH 4/7] sorting string validation --- backend/app/api/v1/commons/ocp.py | 3 +- backend/app/api/v1/commons/quay.py | 3 +- backend/app/api/v1/commons/telco.py | 2 - backend/app/api/v1/commons/utils.py | 27 ++++++++++++ backend/app/api/v1/endpoints/ocp/ocpJobs.py | 41 +------------------ backend/app/api/v1/endpoints/quay/quayJobs.py | 41 +------------------ 6 files changed, 31 insertions(+), 86 deletions(-) diff --git a/backend/app/api/v1/commons/ocp.py b/backend/app/api/v1/commons/ocp.py index db0ed658..51551fbd 100644 --- a/backend/app/api/v1/commons/ocp.py +++ b/backend/app/api/v1/commons/ocp.py @@ -20,8 +20,7 @@ async def getData( }, } if sort: - key, direction = sort.split(":") - query["sort"] = [{key: {"order": direction}}] + query["sort"] = utils.build_sort_terms(sort) es = ElasticService(configpath=configpath) response = await es.post( diff --git a/backend/app/api/v1/commons/quay.py b/backend/app/api/v1/commons/quay.py index b10874d4..45dad74e 100644 --- a/backend/app/api/v1/commons/quay.py +++ b/backend/app/api/v1/commons/quay.py @@ -16,8 +16,7 @@ async def getData( } if sort: - key, direction = sort.split(":") - query["sort"] = [{key: {"order": direction}}] + query["sort"] = utils.build_sort_terms(sort) es = ElasticService(configpath=configpath) response = await es.post( diff --git a/backend/app/api/v1/commons/telco.py b/backend/app/api/v1/commons/telco.py index 91cba718..b55e746f 100644 --- a/backend/app/api/v1/commons/telco.py +++ b/backend/app/api/v1/commons/telco.py @@ -38,8 +38,6 @@ async def getData( "earliest_time": "{}T00:00:00".format(start_datetime.strftime("%Y-%m-%d")), "latest_time": "{}T23:59:59".format(end_datetime.strftime("%Y-%m-%d")), "output_mode": "json", - "sort_dir": "asc", - "sort_key": "test_type", } searchList = " OR ".join( ['test_type="{}"'.format(test_type) for test_type in test_types] diff --git a/backend/app/api/v1/commons/utils.py b/backend/app/api/v1/commons/utils.py index ddcef9d2..1c2deb10 100644 --- a/backend/app/api/v1/commons/utils.py +++ b/backend/app/api/v1/commons/utils.py @@ -1,4 +1,6 @@ from app.services.search import ElasticService +from fastapi import HTTPException, status +import re async def getMetadata(uuid: str, configpath: str): @@ -65,3 +67,28 @@ def getReleaseStream(row): elif row["releaseStream"].__contains__("ec"): return "Engineering Candidate" return "Stable" + + +def build_sort_terms(sort_string: str) -> list[dict[str, str]]: + """ + + Validates and transforms a sort string in the format 'sort=key:direction' to + a list of dictionaries [{key: {"order": direction}}]. + + :param sort_string: str, input string in the format 'sort=key:direction' + + :return: list, transformed sort structure or raises a ValueError for invalid input + + """ + + pattern = r"^([\w]+):(asc|desc)$" + match = re.match(pattern, sort_string) + + if not match: + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + f"Invalid sort string format. Expected 'sort=key:direction' with direction as 'asc' or 'desc'.", + ) + + key, direction = match.groups() + return [{key: {"order": direction}}] diff --git a/backend/app/api/v1/endpoints/ocp/ocpJobs.py b/backend/app/api/v1/endpoints/ocp/ocpJobs.py index f161c50f..c0308ac1 100644 --- a/backend/app/api/v1/endpoints/ocp/ocpJobs.py +++ b/backend/app/api/v1/endpoints/ocp/ocpJobs.py @@ -32,16 +32,10 @@ async def jobs( description="End date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-15"], ), -<<<<<<< HEAD - pretty: bool = Query(False, description="Output content in pretty format."), - size: int = Query(None, description="Number of jobs to fetch"), - offset: int = Query(None, description="Offset Number to fetch jobs from"), -======= pretty: bool = Query(False, description="Output contet in pretty format."), size: int = Query(None, description="Number of jobs to fetch"), offset: int = Query(None, description="Offset Number to fetch jobs from"), - sort: str = Query(None, descption="To sort fields on specified direction"), ->>>>>>> bd597e7 (Sorting) + sort: str = Query(None, description="To sort fields on specified direction"), ): if start_date is None: start_date = datetime.utcnow().date() @@ -58,7 +52,6 @@ async def jobs( status_code=422, ) -<<<<<<< HEAD if offset and not size: raise HTTPException(400, f"offset {offset} specified without size") elif not offset and not size: @@ -79,38 +72,6 @@ async def jobs( "total": results["total"], "offset": offset + size, } -======= - if not offset: - offset = 0 - - if not size: - size = 10000 - offset = 0 - - if not sort: - sort = None - - results = await getData( - start_date, end_date, size, offset, sort, "ocp.elasticsearch" - ) - - if "data" in results and len(results["data"]) >= 1: - response = { - "startDate": start_date.__str__(), - "endDate": end_date.__str__(), - "results": results["data"].to_dict("records"), - "total": results["total"], - "offset": offset + size, - } - else: - response = { - "startDate": start_date.__str__(), - "endDate": end_date.__str__(), - "results": [], - "total": 0, - "offset": 0, - } ->>>>>>> bd597e7 (Sorting) if pretty: json_str = json.dumps(response, indent=4) diff --git a/backend/app/api/v1/endpoints/quay/quayJobs.py b/backend/app/api/v1/endpoints/quay/quayJobs.py index 3a212627..c7c7ce31 100644 --- a/backend/app/api/v1/endpoints/quay/quayJobs.py +++ b/backend/app/api/v1/endpoints/quay/quayJobs.py @@ -32,16 +32,10 @@ async def jobs( description="End date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-15"], ), -<<<<<<< HEAD - pretty: bool = Query(False, description="Output content in pretty format."), - size: int = Query(None, description="Number of jobs to fetch"), - offset: int = Query(None, description="Offset Number to fetch jobs from"), -======= pretty: bool = Query(False, description="Output contet in pretty format."), size: int = Query(None, description="Number of jobs to fetch"), offset: int = Query(None, description="Offset Number to fetch jobs from"), - sort: str = Query(None, descption="To sort fields on specified direction"), ->>>>>>> bd597e7 (Sorting) + sort: str = Query(None, description="To sort fields on specified direction"), ): if start_date is None: start_date = datetime.utcnow().date() @@ -58,7 +52,6 @@ async def jobs( status_code=422, ) -<<<<<<< HEAD if offset and not size: raise HTTPException(400, f"offset {offset} specified without size") elif not offset and not size: @@ -80,38 +73,6 @@ async def jobs( "total": 0, "offset": 0, } -======= - if not offset: - offset = 0 - - if not size: - size = 10000 - offset = 0 - - if not sort: - sort = None - - results = await getData( - start_date, end_date, size, offset, sort, "quay.elasticsearch" - ) - - if "data" in results and len(results["data"]) >= 1: - response = { - "startDate": start_date.__str__(), - "endDate": end_date.__str__(), - "results": results["data"].to_dict("records"), - "total": results["total"], - "offset": offset + size, - } - else: - response = { - "startDate": start_date.__str__(), - "endDate": end_date.__str__(), - "results": [], - "total": 0, - "offset": 0, - } ->>>>>>> bd597e7 (Sorting) if pretty: json_str = json.dumps(response, indent=4) From 0dd7ec68fd3d3c6afee549eb21b032c7c4947b50 Mon Sep 17 00:00:00 2001 From: MVarshini Date: Fri, 13 Dec 2024 22:21:12 +0530 Subject: [PATCH 5/7] sorting string validation --- backend/app/api/v1/commons/constants.py | 23 +++++++++++++++ backend/app/api/v1/commons/utils.py | 28 +++++++++++-------- backend/app/api/v1/endpoints/ocp/ocpJobs.py | 9 ++++-- backend/app/api/v1/endpoints/quay/quayJobs.py | 9 ++++-- backend/app/services/splunk.py | 2 -- frontend/src/actions/commonActions.js | 1 - frontend/src/reducers/ocpReducer.js | 1 - 7 files changed, 52 insertions(+), 21 deletions(-) create mode 100644 backend/app/api/v1/commons/constants.py diff --git a/backend/app/api/v1/commons/constants.py b/backend/app/api/v1/commons/constants.py new file mode 100644 index 00000000..650e9b8c --- /dev/null +++ b/backend/app/api/v1/commons/constants.py @@ -0,0 +1,23 @@ +# Define the keywords for sorting. +DIRECTIONS = ("asc", "desc") +FIELDS = ( + "ciSystem.keyword", + "benchmark.keyword", + "ocpVersion.keyword", + "releaseStream.keyword", + "platform.keyword", + "networkType.keyword", + "ipsec.keyword", + "fips.keyword", + "encrypted.keyword", + "publish.keyword", + "computeArch.keyword", + "controlPlaneArch.keyword", + "jobStatus.keyword", + "startDate", + "endDate", + "workerNodesCount", + "masterNodesCount", + "infraNodesCount", + "totalNodesCount", +) diff --git a/backend/app/api/v1/commons/utils.py b/backend/app/api/v1/commons/utils.py index 1c2deb10..61646f5b 100644 --- a/backend/app/api/v1/commons/utils.py +++ b/backend/app/api/v1/commons/utils.py @@ -1,6 +1,7 @@ from app.services.search import ElasticService from fastapi import HTTPException, status import re +import app.api.v1.commons.constants as constants async def getMetadata(uuid: str, configpath: str): @@ -80,15 +81,18 @@ def build_sort_terms(sort_string: str) -> list[dict[str, str]]: :return: list, transformed sort structure or raises a ValueError for invalid input """ - - pattern = r"^([\w]+):(asc|desc)$" - match = re.match(pattern, sort_string) - - if not match: - raise HTTPException( - status.HTTP_400_BAD_REQUEST, - f"Invalid sort string format. Expected 'sort=key:direction' with direction as 'asc' or 'desc'.", - ) - - key, direction = match.groups() - return [{key: {"order": direction}}] + sort_terms = [] + if sort_string: + key, dir = sort_string.split(":", maxsplit=1) + if dir not in constants.DIRECTIONS: + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + f"Sort direction {dir!r} must be one of {','.join(constants.DIRECTIONS)}", + ) + if key not in constants.FIELDS: + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + f"Sort key {key!r} must be one of {','.join(constants.FIELDS)}", + ) + sort_terms.append({f"{key}": {"order": dir}}) + return sort_terms diff --git a/backend/app/api/v1/endpoints/ocp/ocpJobs.py b/backend/app/api/v1/endpoints/ocp/ocpJobs.py index c0308ac1..b700c600 100644 --- a/backend/app/api/v1/endpoints/ocp/ocpJobs.py +++ b/backend/app/api/v1/endpoints/ocp/ocpJobs.py @@ -12,7 +12,7 @@ @router.get( "/api/v1/ocp/jobs", summary="Returns a job list", - description="Returns a list of jobs in the specified dates of requested size. \ + description="Returns a list of jobs in the specified dates. \ If not dates are provided the API will default the values. \ `startDate`: will be set to the day of the request minus 5 days.\ `endDate`: will be set to the day of the request.", @@ -51,7 +51,8 @@ async def jobs( ), status_code=422, ) - + if not sort: + sort = None if offset and not size: raise HTTPException(400, f"offset {offset} specified without size") elif not offset and not size: @@ -60,7 +61,9 @@ async def jobs( elif not offset: offset = 0 - results = await getData(start_date, end_date, size, offset, "ocp.elasticsearch") + results = await getData( + start_date, end_date, size, offset, sort, "ocp.elasticsearch" + ) jobs = [] if "data" in results and len(results["data"]) >= 1: jobs = results["data"].to_dict("records") diff --git a/backend/app/api/v1/endpoints/quay/quayJobs.py b/backend/app/api/v1/endpoints/quay/quayJobs.py index c7c7ce31..01dca39a 100644 --- a/backend/app/api/v1/endpoints/quay/quayJobs.py +++ b/backend/app/api/v1/endpoints/quay/quayJobs.py @@ -32,7 +32,7 @@ async def jobs( description="End date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-15"], ), - pretty: bool = Query(False, description="Output contet in pretty format."), + pretty: bool = Query(False, description="Output content in pretty format."), size: int = Query(None, description="Number of jobs to fetch"), offset: int = Query(None, description="Offset Number to fetch jobs from"), sort: str = Query(None, description="To sort fields on specified direction"), @@ -60,7 +60,12 @@ async def jobs( elif not offset: offset = 0 - results = await getData(start_date, end_date, size, offset, "quay.elasticsearch") + if not sort: + sort = None + + results = await getData( + start_date, end_date, size, offset, sort, "quay.elasticsearch" + ) jobs = [] if "data" in results and len(results["data"]) >= 1: diff --git a/backend/app/services/splunk.py b/backend/app/services/splunk.py index d7cd8535..70b00263 100644 --- a/backend/app/services/splunk.py +++ b/backend/app/services/splunk.py @@ -45,8 +45,6 @@ async def query( """ query["count"] = size query["offset"] = offset - query["sort_dir"] = "asc" - query["sort_key"] = "test_type" # If additional search parameters are provided, include those in searchindex searchindex = ( diff --git a/frontend/src/actions/commonActions.js b/frontend/src/actions/commonActions.js index 2698a52c..0eec4c94 100644 --- a/frontend/src/actions/commonActions.js +++ b/frontend/src/actions/commonActions.js @@ -141,7 +141,6 @@ export const getSelectedFilter = export const getRequestParams = (type) => (dispatch, getState) => { const { start_date, end_date, size, offset, sort } = getState()[type]; - // const sortParam = `${activeSortIndex}:${activeSortDir}`; const params = { pretty: true, ...(start_date && { start_date }), diff --git a/frontend/src/reducers/ocpReducer.js b/frontend/src/reducers/ocpReducer.js index 78f1df2e..5e93b8b6 100644 --- a/frontend/src/reducers/ocpReducer.js +++ b/frontend/src/reducers/ocpReducer.js @@ -48,7 +48,6 @@ const initialState = { size: DEFAULT_PER_PAGE, offset: INITAL_OFFSET, totalJobs: 0, - //tableData: [], filterData: [], categoryFilterValue: "", filterOptions: [], From ea5f06c8bfe4b7e9eb51e48e69c32a477db193d2 Mon Sep 17 00:00:00 2001 From: MVarshini Date: Fri, 13 Dec 2024 22:42:20 +0530 Subject: [PATCH 6/7] remove sort Str to None --- backend/app/api/v1/endpoints/ocp/ocpJobs.py | 3 +-- backend/app/api/v1/endpoints/quay/quayJobs.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/backend/app/api/v1/endpoints/ocp/ocpJobs.py b/backend/app/api/v1/endpoints/ocp/ocpJobs.py index b700c600..441c4126 100644 --- a/backend/app/api/v1/endpoints/ocp/ocpJobs.py +++ b/backend/app/api/v1/endpoints/ocp/ocpJobs.py @@ -51,8 +51,7 @@ async def jobs( ), status_code=422, ) - if not sort: - sort = None + if offset and not size: raise HTTPException(400, f"offset {offset} specified without size") elif not offset and not size: diff --git a/backend/app/api/v1/endpoints/quay/quayJobs.py b/backend/app/api/v1/endpoints/quay/quayJobs.py index 01dca39a..0a86db56 100644 --- a/backend/app/api/v1/endpoints/quay/quayJobs.py +++ b/backend/app/api/v1/endpoints/quay/quayJobs.py @@ -60,9 +60,6 @@ async def jobs( elif not offset: offset = 0 - if not sort: - sort = None - results = await getData( start_date, end_date, size, offset, sort, "quay.elasticsearch" ) From dbd0c1c583722b25971709c2c98c4513060d3cc5 Mon Sep 17 00:00:00 2001 From: MVarshini Date: Fri, 13 Dec 2024 22:43:29 +0530 Subject: [PATCH 7/7] spell check --- backend/app/api/v1/endpoints/ocp/ocpJobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app/api/v1/endpoints/ocp/ocpJobs.py b/backend/app/api/v1/endpoints/ocp/ocpJobs.py index 441c4126..07b689a5 100644 --- a/backend/app/api/v1/endpoints/ocp/ocpJobs.py +++ b/backend/app/api/v1/endpoints/ocp/ocpJobs.py @@ -32,7 +32,7 @@ async def jobs( description="End date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-15"], ), - pretty: bool = Query(False, description="Output contet in pretty format."), + pretty: bool = Query(False, description="Output content in pretty format."), size: int = Query(None, description="Number of jobs to fetch"), offset: int = Query(None, description="Offset Number to fetch jobs from"), sort: str = Query(None, description="To sort fields on specified direction"),