Skip to content

Commit

Permalink
Merge pull request Sunbird-Obsrv#157 from shikshalokam/release-4.10.5
Browse files Browse the repository at this point in the history 
SB-30373 - Sort Issue Fix
  • Loading branch information
@sowmya-dixit
sowmya-dixit authored Jun 29, 2022
2 parents 403c8ef + 44b503d commit 2fa5000
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 6 deletions.
8 changes: 4 additions & 4 deletions batch-models/src/main/scala/org/ekstep/analytics/exhaust/OnDemandDruidExhaustJob.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,16 +100,16 @@ object OnDemandDruidExhaustJob extends BaseReportsJob with Serializable with IJo
(df.columns).map(f1 => {
df = df.withColumn(f1, when((col(f1) === "unknown") || (col(f1) === "<NULL>"), "Null").otherwise(col(f1)))
})
if (sortDfColNames.isDefined) {
df = df.sort(sortDfColNames.get.head, sortDfColNames.get.tail: _*)
}
df = df.dropDuplicates()
if (dataCount.value > 0) {
val metricFields = f.metrics
val fieldsList = (dimFields ++ metricFields).distinct
val dimsLabels = labelsLookup.filter(x => f.dims.contains(x._1)).values.toList
val filteredDf = df.select(fieldsList.head, fieldsList.tail: _*)
val renamedDf = filteredDf.select(filteredDf.columns.map(c => filteredDf.col(c).as(labelsLookup.getOrElse(c, c))): _*).na.fill("unknown")
var renamedDf = filteredDf.select(filteredDf.columns.map(c => filteredDf.col(c).as(labelsLookup.getOrElse(c, c))): _*).na.fill("unknown")
if (sortDfColNames.isDefined) {
renamedDf = renamedDf.sort(sortDfColNames.get.head, sortDfColNames.get.tail: _*)
}
val reportFinalId = reportConfig.id + "/" + request_id + "_" + reportDate
fileSavedToBlob = saveReport(renamedDf, JSONUtils.deserialize[Map[String, AnyRef]](JSONUtils.serialize(config.modelParams.get)) ++
Map("dims" -> dimsLabels, "reportId" -> reportFinalId, "fileParameters" -> f.fileParameters, "format" -> f.`type`), None, None)
Expand Down
4 changes: 2 additions & 2 deletions batch-models/src/test/resources/application.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,11 +157,11 @@ postgres.table.job_request="job_request"
postgres.table.dataset_metadata="dataset_metadata"

druid_query = {
ml-task-detail-exhaust = """{"id":"ml-task-detail-exhaust","labels":{"block_name":"Block","project_title_editable":"Project Title","task_evidence":"Evidence","designation":"User sub type","school_externalId":"School ID","project_duration":"Project Duration","status_of_project":"Project Status","sub_task":"Sub-Tasks","tasks":"Tasks","project_id":"Project ID","project_description":"Project Objective","program_externalId":"Program ID","organisation_name":"Org Name","createdBy":"UUID","area_of_improvement":"Category","school_name":"School Name","district_name":"District","program_name":"Program Name","state_name":"Declared State","task_remarks":"Remarks"},"dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","dataSource":"sl-project","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","status_of_project","tasks","sub_task","task_evidence","task_remarks"],"queryType":"scan"}}],"output":[{"zip":false,"label":"","dims":["date"],"fileParameters":["id","dims"],"metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","status_of_project","tasks","sub_task","task_evidence","task_remarks"],"type":"csv"}],"queryType":"scan","sort":["createdBy","program_externalId","project_id","tasks"]}"""
ml-task-detail-exhaust = """{"id":"ml-task-detail-exhaust","labels":{"block_name":"Block","project_title_editable":"Project Title","task_evidence":"Evidence","designation":"User sub type","school_externalId":"School ID","project_duration":"Project Duration","status_of_project":"Project Status","sub_task":"Sub-Tasks","tasks":"Tasks","project_id":"Project ID","project_description":"Project Objective","program_externalId":"Program ID","organisation_name":"Org Name","createdBy":"UUID","area_of_improvement":"Category","school_name":"School Name","district_name":"District","program_name":"Program Name","state_name":"Declared State","task_remarks":"Remarks"},"dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","dataSource":"sl-project","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","status_of_project","tasks","sub_task","task_evidence","task_remarks"],"queryType":"scan"}}],"output":[{"zip":false,"label":"","dims":["date"],"fileParameters":["id","dims"],"metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","status_of_project","tasks","sub_task","task_evidence","task_remarks"],"type":"csv"}],"queryType":"scan","sort":["UUID","Program ID","Project ID","Tasks"]}"""
ml-task-detail-exhaust-no-range = """{"id":"ml-task-detail-exhaust","queryType":"scan","dateRange":{"staticInterval":null,"granularity":null},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"queryType":"scan","dataSource":"sl-project","intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"]}}],"labels":{"createdBy":"UUID","designation":"Role","state_name":"Declared State","district_name":"District","block_name":"Block","school_name":"School Name","school_externalId":"School ID","organisation_name":"Organisation Name","program_name":"Program Name","program_externalId":"Program ID","project_id":"Project ID","project_title_editable":"Project Title","project_description":"Project Objective","area_of_improvement":"Category","project_duration":"Project Duration","tasks":"Tasks","sub_task":"Sub-Tasks","task_evidence":"Evidence","task_remarks":"Remarks","status_of_project":"Project Status"},"output":[{"type":"csv","metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"],"fileParameters":["id","dims"],"zip":true,"dims":["date"],"label":""}]}"""
ml-task-detail-exhaust-no-interval = """{"id":"ml-task-detail-exhaust","queryType":"scan","dateRange":{"interval":null,"granularity":null,"intervalSlider":0},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"queryType":"scan","dataSource":"sl-project","intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"]}}],"labels":{"createdBy":"UUID","designation":"Role","state_name":"Declared State","district_name":"District","block_name":"Block","school_name":"School Name","school_externalId":"School ID","organisation_name":"Organisation Name","program_name":"Program Name","program_externalId":"Program ID","project_id":"Project ID","project_title_editable":"Project Title","project_description":"Project Objective","area_of_improvement":"Category","project_duration":"Project Duration","tasks":"Tasks","sub_task":"Sub-Tasks","task_evidence":"Evidence","task_remarks":"Remarks","status_of_project":"Project Status"},"output":[{"type":"csv","metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"],"fileParameters":["id","dims"],"zip":false,"dims":["date"],"label":""}]}"""
ml-task-detail-exhaust-static-interval ="""{"id":"ml-task-detail-exhaust","queryType":"scan","dateRange":{"interval":{"startDate":"1901-01-01","endDate":"2101-01-01"},"granularity":null,"intervalSlider":0},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"queryType":"scan","dataSource":"sl-project","intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"]}}],"labels":{"createdBy":"UUID","designation":"Role","state_name":"Declared State","district_name":"District","block_name":"Block","school_name":"School Name","school_externalId":"School ID","organisation_name":"Organisation Name","program_name":"Program Name","program_externalId":"Program ID","project_id":"Project ID","project_title_editable":"Project Title","project_description":"Project Objective","area_of_improvement":"Category","project_duration":"Project Duration","tasks":"Tasks","sub_task":"Sub-Tasks","task_evidence":"Evidence","task_remarks":"Remarks","status_of_project":"Project Status"},"output":[{"type":"csv","metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"],"fileParameters":["id","dims"],"zip":false,"dims":["date"],"label":""}]}"""
ml-task-detail-exhaust-quote-column ="""{"id":"ml-task-detail-exhaust","queryType":"scan","dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"queryType":"scan","dataSource":"sl-project","intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"]}}],"labels":{"createdBy":"UUID","designation":"Role","state_name":"Declared State","district_name":"District","block_name":"Block","school_name":"School Name","school_externalId":"School ID","organisation_name":"Organisation Name","program_name":"Program Name","program_externalId":"Program ID","project_id":"Project ID","project_title_editable":"Project Title","project_description":"Project Objective","area_of_improvement":"Category","project_duration":"Project Duration","tasks":"Tasks","sub_task":"Sub-Tasks","task_evidence":"Evidence","task_remarks":"Remarks","status_of_project":"Project Status"},"output":[{"type":"csv","metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"],"fileParameters":["id","dims"],"zip":false,"dims":["date"],"label":""}]}"""
ml-task-detail-exhaust-no-label = """{"id":"ml-task-detail-exhaust","queryType":"scan","dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[],"labels":{},"output":[{"type":"csv","metrics":[],"fileParameters":["id","dims"],"zip":false,"dims":["date"],"label":""}]}"""
ml-obs-question-detail-exhaust = """{"id":"ml-obs-question-detail-exhaust","labels":{"questionName":"Question","user_districtName":"District","evidences":"Evidences","questionResponseLabel":"Question_response_label","solutionExternalId":"Observation ID","user_schoolUDISE_code":"School ID","role_title":"User Sub Type","minScore":"Question score","programName":"Program Name","questionExternalId":"Question_external_id","organisation_name":"Organisation Name","createdBy":"UUID","remarks":"Remarks","user_blockName":"Block","solutionName":"Observation Name","user_schoolName":"School Name","programExternalId":"Program ID","user_stateName":"Declared State","observationSubmissionId":"observation_submission_id"},"dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","dataSource":"sl-observation","columns":["createdBy","role_title","user_stateName","user_districtName","user_blockName","user_schoolUDISE_code","user_schoolName","organisation_name","programName","programExternalId","solutionName","solutionExternalId","observationSubmissionId","questionExternalId","questionName","questionResponseLabel","minScore","evidences","remarks"],"queryType":"scan"}}],"output":[{"zip":false,"label":"","dims":["date"],"fileParameters":["id","dims"],"metrics":["createdBy","role_title","user_stateName","user_districtName","user_blockName","user_schoolUDISE_code","user_schoolName","organisation_name","programName","programExternalId","solutionName","solutionExternalId","observationSubmissionId","questionExternalId","questionName","questionResponseLabel","minScore","evidences","remarks"],"type":"csv"}],"queryType":"scan","sort":["createdBy","programExternalId","solutionExternalId","observationSubmissionId","questionExternalId"]}"""
ml-obs-question-detail-exhaust = """{"id":"ml-obs-question-detail-exhaust","labels":{"questionName":"Question","user_districtName":"District","evidences":"Evidences","questionResponseLabel":"Question_response_label","solutionExternalId":"Observation ID","user_schoolUDISE_code":"School ID","role_title":"User Sub Type","minScore":"Question score","programName":"Program Name","questionExternalId":"Question_external_id","organisation_name":"Organisation Name","createdBy":"UUID","remarks":"Remarks","user_blockName":"Block","solutionName":"Observation Name","user_schoolName":"School Name","programExternalId":"Program ID","user_stateName":"Declared State","observationSubmissionId":"observation_submission_id"},"dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","dataSource":"sl-observation","columns":["createdBy","role_title","user_stateName","user_districtName","user_blockName","user_schoolUDISE_code","user_schoolName","organisation_name","programName","programExternalId","solutionName","solutionExternalId","observationSubmissionId","questionExternalId","questionName","questionResponseLabel","minScore","evidences","remarks"],"queryType":"scan"}}],"output":[{"zip":false,"label":"","dims":["date"],"fileParameters":["id","dims"],"metrics":["createdBy","role_title","user_stateName","user_districtName","user_blockName","user_schoolUDISE_code","user_schoolName","organisation_name","programName","programExternalId","solutionName","solutionExternalId","observationSubmissionId","questionExternalId","questionName","questionResponseLabel","minScore","evidences","remarks"],"type":"csv"}],"queryType":"scan","sort":["UUID","Program ID","Observation ID","observation_submission_id","Question_external_id"]}"""
}

0 comments on commit 2fa5000

Please sign in to comment.