From 0fa5cd12366f57e8f424dcddb1aebee48fdc6850 Mon Sep 17 00:00:00 2001 From: Shakthieshwari Date: Wed, 29 Jun 2022 18:00:22 +0530 Subject: [PATCH 1/4] Sort Issue Fix --- .../analytics/exhaust/OnDemandDruidExhaustJob.scala | 8 ++++---- batch-models/src/test/resources/application.conf | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/batch-models/src/main/scala/org/ekstep/analytics/exhaust/OnDemandDruidExhaustJob.scala b/batch-models/src/main/scala/org/ekstep/analytics/exhaust/OnDemandDruidExhaustJob.scala index 85bb3bae..cbc0ef47 100644 --- a/batch-models/src/main/scala/org/ekstep/analytics/exhaust/OnDemandDruidExhaustJob.scala +++ b/batch-models/src/main/scala/org/ekstep/analytics/exhaust/OnDemandDruidExhaustJob.scala @@ -100,16 +100,16 @@ object OnDemandDruidExhaustJob extends BaseReportsJob with Serializable with IJo (df.columns).map(f1 => { df = df.withColumn(f1, when((col(f1) === "unknown") || (col(f1) === ""), "Null").otherwise(col(f1))) }) - if (sortDfColNames.isDefined) { - df = df.sort(sortDfColNames.get.head, sortDfColNames.get.tail: _*) - } df = df.dropDuplicates() if (dataCount.value > 0) { val metricFields = f.metrics val fieldsList = (dimFields ++ metricFields).distinct val dimsLabels = labelsLookup.filter(x => f.dims.contains(x._1)).values.toList val filteredDf = df.select(fieldsList.head, fieldsList.tail: _*) - val renamedDf = filteredDf.select(filteredDf.columns.map(c => filteredDf.col(c).as(labelsLookup.getOrElse(c, c))): _*).na.fill("unknown") + var renamedDf = filteredDf.select(filteredDf.columns.map(c => filteredDf.col(c).as(labelsLookup.getOrElse(c, c))): _*).na.fill("unknown") + if (sortDfColNames.isDefined) { + renamedDf = renamedDf.sort(sortDfColNames.get.head, sortDfColNames.get.tail: _*) + } val reportFinalId = reportConfig.id + "/" + request_id + "_" + reportDate fileSavedToBlob = saveReport(renamedDf, JSONUtils.deserialize[Map[String, AnyRef]](JSONUtils.serialize(config.modelParams.get)) ++ Map("dims" -> dimsLabels, "reportId" -> reportFinalId, "fileParameters" -> f.fileParameters, "format" -> f.`type`), None, None) diff --git a/batch-models/src/test/resources/application.conf b/batch-models/src/test/resources/application.conf index 286bf388..5dd59080 100644 --- a/batch-models/src/test/resources/application.conf +++ b/batch-models/src/test/resources/application.conf @@ -157,11 +157,11 @@ postgres.table.job_request="job_request" postgres.table.dataset_metadata="dataset_metadata" druid_query = { - ml-task-detail-exhaust = """{"id":"ml-task-detail-exhaust","labels":{"block_name":"Block","project_title_editable":"Project Title","task_evidence":"Evidence","designation":"User sub type","school_externalId":"School ID","project_duration":"Project Duration","status_of_project":"Project Status","sub_task":"Sub-Tasks","tasks":"Tasks","project_id":"Project ID","project_description":"Project Objective","program_externalId":"Program ID","organisation_name":"Org Name","createdBy":"UUID","area_of_improvement":"Category","school_name":"School Name","district_name":"District","program_name":"Program Name","state_name":"Declared State","task_remarks":"Remarks"},"dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","dataSource":"sl-project","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","status_of_project","tasks","sub_task","task_evidence","task_remarks"],"queryType":"scan"}}],"output":[{"zip":false,"label":"","dims":["date"],"fileParameters":["id","dims"],"metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","status_of_project","tasks","sub_task","task_evidence","task_remarks"],"type":"csv"}],"queryType":"scan","sort":["createdBy","program_externalId","project_id","tasks"]}""" + ml-task-detail-exhaust = """{"id":"ml-task-detail-exhaust","labels":{"block_name":"Block","project_title_editable":"Project Title","task_evidence":"Evidence","designation":"User sub type","school_externalId":"School ID","project_duration":"Project Duration","status_of_project":"Project Status","sub_task":"Sub-Tasks","tasks":"Tasks","project_id":"Project ID","project_description":"Project Objective","program_externalId":"Program ID","organisation_name":"Org Name","createdBy":"UUID","area_of_improvement":"Category","school_name":"School Name","district_name":"District","program_name":"Program Name","state_name":"Declared State","task_remarks":"Remarks"},"dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","dataSource":"sl-project","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","status_of_project","tasks","sub_task","task_evidence","task_remarks"],"queryType":"scan"}}],"output":[{"zip":false,"label":"","dims":["date"],"fileParameters":["id","dims"],"metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","status_of_project","tasks","sub_task","task_evidence","task_remarks"],"type":"csv"}],"queryType":"scan","sort":["UUID","Program ID","Project ID","Tasks"]}""" ml-task-detail-exhaust-no-range = """{"id":"ml-task-detail-exhaust","queryType":"scan","dateRange":{"staticInterval":null,"granularity":null},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"queryType":"scan","dataSource":"sl-project","intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"]}}],"labels":{"createdBy":"UUID","designation":"Role","state_name":"Declared State","district_name":"District","block_name":"Block","school_name":"School Name","school_externalId":"School ID","organisation_name":"Organisation Name","program_name":"Program Name","program_externalId":"Program ID","project_id":"Project ID","project_title_editable":"Project Title","project_description":"Project Objective","area_of_improvement":"Category","project_duration":"Project Duration","tasks":"Tasks","sub_task":"Sub-Tasks","task_evidence":"Evidence","task_remarks":"Remarks","status_of_project":"Project Status"},"output":[{"type":"csv","metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"],"fileParameters":["id","dims"],"zip":true,"dims":["date"],"label":""}]}""" ml-task-detail-exhaust-no-interval = """{"id":"ml-task-detail-exhaust","queryType":"scan","dateRange":{"interval":null,"granularity":null,"intervalSlider":0},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"queryType":"scan","dataSource":"sl-project","intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"]}}],"labels":{"createdBy":"UUID","designation":"Role","state_name":"Declared State","district_name":"District","block_name":"Block","school_name":"School Name","school_externalId":"School ID","organisation_name":"Organisation Name","program_name":"Program Name","program_externalId":"Program ID","project_id":"Project ID","project_title_editable":"Project Title","project_description":"Project Objective","area_of_improvement":"Category","project_duration":"Project Duration","tasks":"Tasks","sub_task":"Sub-Tasks","task_evidence":"Evidence","task_remarks":"Remarks","status_of_project":"Project Status"},"output":[{"type":"csv","metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"],"fileParameters":["id","dims"],"zip":false,"dims":["date"],"label":""}]}""" ml-task-detail-exhaust-static-interval ="""{"id":"ml-task-detail-exhaust","queryType":"scan","dateRange":{"interval":{"startDate":"1901-01-01","endDate":"2101-01-01"},"granularity":null,"intervalSlider":0},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"queryType":"scan","dataSource":"sl-project","intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"]}}],"labels":{"createdBy":"UUID","designation":"Role","state_name":"Declared State","district_name":"District","block_name":"Block","school_name":"School Name","school_externalId":"School ID","organisation_name":"Organisation Name","program_name":"Program Name","program_externalId":"Program ID","project_id":"Project ID","project_title_editable":"Project Title","project_description":"Project Objective","area_of_improvement":"Category","project_duration":"Project Duration","tasks":"Tasks","sub_task":"Sub-Tasks","task_evidence":"Evidence","task_remarks":"Remarks","status_of_project":"Project Status"},"output":[{"type":"csv","metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"],"fileParameters":["id","dims"],"zip":false,"dims":["date"],"label":""}]}""" ml-task-detail-exhaust-quote-column ="""{"id":"ml-task-detail-exhaust","queryType":"scan","dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"queryType":"scan","dataSource":"sl-project","intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"]}}],"labels":{"createdBy":"UUID","designation":"Role","state_name":"Declared State","district_name":"District","block_name":"Block","school_name":"School Name","school_externalId":"School ID","organisation_name":"Organisation Name","program_name":"Program Name","program_externalId":"Program ID","project_id":"Project ID","project_title_editable":"Project Title","project_description":"Project Objective","area_of_improvement":"Category","project_duration":"Project Duration","tasks":"Tasks","sub_task":"Sub-Tasks","task_evidence":"Evidence","task_remarks":"Remarks","status_of_project":"Project Status"},"output":[{"type":"csv","metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"],"fileParameters":["id","dims"],"zip":false,"dims":["date"],"label":""}]}""" ml-task-detail-exhaust-no-label = """{"id":"ml-task-detail-exhaust","queryType":"scan","dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[],"labels":{},"output":[{"type":"csv","metrics":[],"fileParameters":["id","dims"],"zip":false,"dims":["date"],"label":""}]}""" - ml-obs-question-detail-exhaust = """{"id":"ml-obs-question-detail-exhaust","labels":{"questionName":"Question","user_districtName":"District","evidences":"Evidences","questionResponseLabel":"Question_response_label","solutionExternalId":"Observation ID","user_schoolUDISE_code":"School ID","role_title":"User Sub Type","minScore":"Question score","programName":"Program Name","questionExternalId":"Question_external_id","organisation_name":"Organisation Name","createdBy":"UUID","remarks":"Remarks","user_blockName":"Block","solutionName":"Observation Name","user_schoolName":"School Name","programExternalId":"Program ID","user_stateName":"Declared State","observationSubmissionId":"observation_submission_id"},"dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","dataSource":"sl-observation","columns":["createdBy","role_title","user_stateName","user_districtName","user_blockName","user_schoolUDISE_code","user_schoolName","organisation_name","programName","programExternalId","solutionName","solutionExternalId","observationSubmissionId","questionExternalId","questionName","questionResponseLabel","minScore","evidences","remarks"],"queryType":"scan"}}],"output":[{"zip":false,"label":"","dims":["date"],"fileParameters":["id","dims"],"metrics":["createdBy","role_title","user_stateName","user_districtName","user_blockName","user_schoolUDISE_code","user_schoolName","organisation_name","programName","programExternalId","solutionName","solutionExternalId","observationSubmissionId","questionExternalId","questionName","questionResponseLabel","minScore","evidences","remarks"],"type":"csv"}],"queryType":"scan","sort":["createdBy","programExternalId","solutionExternalId","observationSubmissionId","questionExternalId"]}""" + ml-obs-question-detail-exhaust = """{"id":"ml-obs-question-detail-exhaust","labels":{"questionName":"Question","user_districtName":"District","evidences":"Evidences","questionResponseLabel":"Question_response_label","solutionExternalId":"Observation ID","user_schoolUDISE_code":"School ID","role_title":"User Sub Type","minScore":"Question score","programName":"Program Name","questionExternalId":"Question_external_id","organisation_name":"Organisation Name","createdBy":"UUID","remarks":"Remarks","user_blockName":"Block","solutionName":"Observation Name","user_schoolName":"School Name","programExternalId":"Program ID","user_stateName":"Declared State","observationSubmissionId":"observation_submission_id"},"dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","dataSource":"sl-observation","columns":["createdBy","role_title","user_stateName","user_districtName","user_blockName","user_schoolUDISE_code","user_schoolName","organisation_name","programName","programExternalId","solutionName","solutionExternalId","observationSubmissionId","questionExternalId","questionName","questionResponseLabel","minScore","evidences","remarks"],"queryType":"scan"}}],"output":[{"zip":false,"label":"","dims":["date"],"fileParameters":["id","dims"],"metrics":["createdBy","role_title","user_stateName","user_districtName","user_blockName","user_schoolUDISE_code","user_schoolName","organisation_name","programName","programExternalId","solutionName","solutionExternalId","observationSubmissionId","questionExternalId","questionName","questionResponseLabel","minScore","evidences","remarks"],"type":"csv"}],"queryType":"scan","sort":["UUID","Program ID","Observation ID","observation_submission_id","Question_external_id"]}""" } \ No newline at end of file From ba3f1c7a2b75895f3d99a2b389d2eed129f656fa Mon Sep 17 00:00:00 2001 From: Shakthieshwari Date: Wed, 29 Jun 2022 18:34:41 +0530 Subject: [PATCH 2/4] Sort Issue Fix --- .../analytics/exhaust/OnDemandDruidExhaustJob.scala | 8 ++++---- batch-models/src/test/resources/application.conf | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/batch-models/src/main/scala/org/ekstep/analytics/exhaust/OnDemandDruidExhaustJob.scala b/batch-models/src/main/scala/org/ekstep/analytics/exhaust/OnDemandDruidExhaustJob.scala index 85bb3bae..cbc0ef47 100644 --- a/batch-models/src/main/scala/org/ekstep/analytics/exhaust/OnDemandDruidExhaustJob.scala +++ b/batch-models/src/main/scala/org/ekstep/analytics/exhaust/OnDemandDruidExhaustJob.scala @@ -100,16 +100,16 @@ object OnDemandDruidExhaustJob extends BaseReportsJob with Serializable with IJo (df.columns).map(f1 => { df = df.withColumn(f1, when((col(f1) === "unknown") || (col(f1) === ""), "Null").otherwise(col(f1))) }) - if (sortDfColNames.isDefined) { - df = df.sort(sortDfColNames.get.head, sortDfColNames.get.tail: _*) - } df = df.dropDuplicates() if (dataCount.value > 0) { val metricFields = f.metrics val fieldsList = (dimFields ++ metricFields).distinct val dimsLabels = labelsLookup.filter(x => f.dims.contains(x._1)).values.toList val filteredDf = df.select(fieldsList.head, fieldsList.tail: _*) - val renamedDf = filteredDf.select(filteredDf.columns.map(c => filteredDf.col(c).as(labelsLookup.getOrElse(c, c))): _*).na.fill("unknown") + var renamedDf = filteredDf.select(filteredDf.columns.map(c => filteredDf.col(c).as(labelsLookup.getOrElse(c, c))): _*).na.fill("unknown") + if (sortDfColNames.isDefined) { + renamedDf = renamedDf.sort(sortDfColNames.get.head, sortDfColNames.get.tail: _*) + } val reportFinalId = reportConfig.id + "/" + request_id + "_" + reportDate fileSavedToBlob = saveReport(renamedDf, JSONUtils.deserialize[Map[String, AnyRef]](JSONUtils.serialize(config.modelParams.get)) ++ Map("dims" -> dimsLabels, "reportId" -> reportFinalId, "fileParameters" -> f.fileParameters, "format" -> f.`type`), None, None) diff --git a/batch-models/src/test/resources/application.conf b/batch-models/src/test/resources/application.conf index 286bf388..5dd59080 100644 --- a/batch-models/src/test/resources/application.conf +++ b/batch-models/src/test/resources/application.conf @@ -157,11 +157,11 @@ postgres.table.job_request="job_request" postgres.table.dataset_metadata="dataset_metadata" druid_query = { - ml-task-detail-exhaust = """{"id":"ml-task-detail-exhaust","labels":{"block_name":"Block","project_title_editable":"Project Title","task_evidence":"Evidence","designation":"User sub type","school_externalId":"School ID","project_duration":"Project Duration","status_of_project":"Project Status","sub_task":"Sub-Tasks","tasks":"Tasks","project_id":"Project ID","project_description":"Project Objective","program_externalId":"Program ID","organisation_name":"Org Name","createdBy":"UUID","area_of_improvement":"Category","school_name":"School Name","district_name":"District","program_name":"Program Name","state_name":"Declared State","task_remarks":"Remarks"},"dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","dataSource":"sl-project","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","status_of_project","tasks","sub_task","task_evidence","task_remarks"],"queryType":"scan"}}],"output":[{"zip":false,"label":"","dims":["date"],"fileParameters":["id","dims"],"metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","status_of_project","tasks","sub_task","task_evidence","task_remarks"],"type":"csv"}],"queryType":"scan","sort":["createdBy","program_externalId","project_id","tasks"]}""" + ml-task-detail-exhaust = """{"id":"ml-task-detail-exhaust","labels":{"block_name":"Block","project_title_editable":"Project Title","task_evidence":"Evidence","designation":"User sub type","school_externalId":"School ID","project_duration":"Project Duration","status_of_project":"Project Status","sub_task":"Sub-Tasks","tasks":"Tasks","project_id":"Project ID","project_description":"Project Objective","program_externalId":"Program ID","organisation_name":"Org Name","createdBy":"UUID","area_of_improvement":"Category","school_name":"School Name","district_name":"District","program_name":"Program Name","state_name":"Declared State","task_remarks":"Remarks"},"dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","dataSource":"sl-project","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","status_of_project","tasks","sub_task","task_evidence","task_remarks"],"queryType":"scan"}}],"output":[{"zip":false,"label":"","dims":["date"],"fileParameters":["id","dims"],"metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","status_of_project","tasks","sub_task","task_evidence","task_remarks"],"type":"csv"}],"queryType":"scan","sort":["UUID","Program ID","Project ID","Tasks"]}""" ml-task-detail-exhaust-no-range = """{"id":"ml-task-detail-exhaust","queryType":"scan","dateRange":{"staticInterval":null,"granularity":null},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"queryType":"scan","dataSource":"sl-project","intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"]}}],"labels":{"createdBy":"UUID","designation":"Role","state_name":"Declared State","district_name":"District","block_name":"Block","school_name":"School Name","school_externalId":"School ID","organisation_name":"Organisation Name","program_name":"Program Name","program_externalId":"Program ID","project_id":"Project ID","project_title_editable":"Project Title","project_description":"Project Objective","area_of_improvement":"Category","project_duration":"Project Duration","tasks":"Tasks","sub_task":"Sub-Tasks","task_evidence":"Evidence","task_remarks":"Remarks","status_of_project":"Project Status"},"output":[{"type":"csv","metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"],"fileParameters":["id","dims"],"zip":true,"dims":["date"],"label":""}]}""" ml-task-detail-exhaust-no-interval = """{"id":"ml-task-detail-exhaust","queryType":"scan","dateRange":{"interval":null,"granularity":null,"intervalSlider":0},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"queryType":"scan","dataSource":"sl-project","intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"]}}],"labels":{"createdBy":"UUID","designation":"Role","state_name":"Declared State","district_name":"District","block_name":"Block","school_name":"School Name","school_externalId":"School ID","organisation_name":"Organisation Name","program_name":"Program Name","program_externalId":"Program ID","project_id":"Project ID","project_title_editable":"Project Title","project_description":"Project Objective","area_of_improvement":"Category","project_duration":"Project Duration","tasks":"Tasks","sub_task":"Sub-Tasks","task_evidence":"Evidence","task_remarks":"Remarks","status_of_project":"Project Status"},"output":[{"type":"csv","metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"],"fileParameters":["id","dims"],"zip":false,"dims":["date"],"label":""}]}""" ml-task-detail-exhaust-static-interval ="""{"id":"ml-task-detail-exhaust","queryType":"scan","dateRange":{"interval":{"startDate":"1901-01-01","endDate":"2101-01-01"},"granularity":null,"intervalSlider":0},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"queryType":"scan","dataSource":"sl-project","intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"]}}],"labels":{"createdBy":"UUID","designation":"Role","state_name":"Declared State","district_name":"District","block_name":"Block","school_name":"School Name","school_externalId":"School ID","organisation_name":"Organisation Name","program_name":"Program Name","program_externalId":"Program ID","project_id":"Project ID","project_title_editable":"Project Title","project_description":"Project Objective","area_of_improvement":"Category","project_duration":"Project Duration","tasks":"Tasks","sub_task":"Sub-Tasks","task_evidence":"Evidence","task_remarks":"Remarks","status_of_project":"Project Status"},"output":[{"type":"csv","metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"],"fileParameters":["id","dims"],"zip":false,"dims":["date"],"label":""}]}""" ml-task-detail-exhaust-quote-column ="""{"id":"ml-task-detail-exhaust","queryType":"scan","dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"queryType":"scan","dataSource":"sl-project","intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","columns":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"]}}],"labels":{"createdBy":"UUID","designation":"Role","state_name":"Declared State","district_name":"District","block_name":"Block","school_name":"School Name","school_externalId":"School ID","organisation_name":"Organisation Name","program_name":"Program Name","program_externalId":"Program ID","project_id":"Project ID","project_title_editable":"Project Title","project_description":"Project Objective","area_of_improvement":"Category","project_duration":"Project Duration","tasks":"Tasks","sub_task":"Sub-Tasks","task_evidence":"Evidence","task_remarks":"Remarks","status_of_project":"Project Status"},"output":[{"type":"csv","metrics":["createdBy","designation","state_name","district_name","block_name","school_name","school_externalId","organisation_name","program_name","program_externalId","project_id","project_title_editable","project_description","area_of_improvement","project_duration","tasks","sub_task","task_evidence","task_remarks","status_of_project"],"fileParameters":["id","dims"],"zip":false,"dims":["date"],"label":""}]}""" ml-task-detail-exhaust-no-label = """{"id":"ml-task-detail-exhaust","queryType":"scan","dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[],"labels":{},"output":[{"type":"csv","metrics":[],"fileParameters":["id","dims"],"zip":false,"dims":["date"],"label":""}]}""" - ml-obs-question-detail-exhaust = """{"id":"ml-obs-question-detail-exhaust","labels":{"questionName":"Question","user_districtName":"District","evidences":"Evidences","questionResponseLabel":"Question_response_label","solutionExternalId":"Observation ID","user_schoolUDISE_code":"School ID","role_title":"User Sub Type","minScore":"Question score","programName":"Program Name","questionExternalId":"Question_external_id","organisation_name":"Organisation Name","createdBy":"UUID","remarks":"Remarks","user_blockName":"Block","solutionName":"Observation Name","user_schoolName":"School Name","programExternalId":"Program ID","user_stateName":"Declared State","observationSubmissionId":"observation_submission_id"},"dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","dataSource":"sl-observation","columns":["createdBy","role_title","user_stateName","user_districtName","user_blockName","user_schoolUDISE_code","user_schoolName","organisation_name","programName","programExternalId","solutionName","solutionExternalId","observationSubmissionId","questionExternalId","questionName","questionResponseLabel","minScore","evidences","remarks"],"queryType":"scan"}}],"output":[{"zip":false,"label":"","dims":["date"],"fileParameters":["id","dims"],"metrics":["createdBy","role_title","user_stateName","user_districtName","user_blockName","user_schoolUDISE_code","user_schoolName","organisation_name","programName","programExternalId","solutionName","solutionExternalId","observationSubmissionId","questionExternalId","questionName","questionResponseLabel","minScore","evidences","remarks"],"type":"csv"}],"queryType":"scan","sort":["createdBy","programExternalId","solutionExternalId","observationSubmissionId","questionExternalId"]}""" + ml-obs-question-detail-exhaust = """{"id":"ml-obs-question-detail-exhaust","labels":{"questionName":"Question","user_districtName":"District","evidences":"Evidences","questionResponseLabel":"Question_response_label","solutionExternalId":"Observation ID","user_schoolUDISE_code":"School ID","role_title":"User Sub Type","minScore":"Question score","programName":"Program Name","questionExternalId":"Question_external_id","organisation_name":"Organisation Name","createdBy":"UUID","remarks":"Remarks","user_blockName":"Block","solutionName":"Observation Name","user_schoolName":"School Name","programExternalId":"Program ID","user_stateName":"Declared State","observationSubmissionId":"observation_submission_id"},"dateRange":{"staticInterval":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all"},"metrics":[{"metric":"total_content_plays_on_portal","label":"total_content_plays_on_portal","druidQuery":{"intervals":"1901-01-01T00:00+00:00/2101-01-01T00:00:00+00:00","dataSource":"sl-observation","columns":["createdBy","role_title","user_stateName","user_districtName","user_blockName","user_schoolUDISE_code","user_schoolName","organisation_name","programName","programExternalId","solutionName","solutionExternalId","observationSubmissionId","questionExternalId","questionName","questionResponseLabel","minScore","evidences","remarks"],"queryType":"scan"}}],"output":[{"zip":false,"label":"","dims":["date"],"fileParameters":["id","dims"],"metrics":["createdBy","role_title","user_stateName","user_districtName","user_blockName","user_schoolUDISE_code","user_schoolName","organisation_name","programName","programExternalId","solutionName","solutionExternalId","observationSubmissionId","questionExternalId","questionName","questionResponseLabel","minScore","evidences","remarks"],"type":"csv"}],"queryType":"scan","sort":["UUID","Program ID","Observation ID","observation_submission_id","Question_external_id"]}""" } \ No newline at end of file From 9b9994ef04973e1e38a5ea6e1ccde316a24bda7b Mon Sep 17 00:00:00 2001 From: Shakthieshwari Date: Wed, 29 Jun 2022 18:57:26 +0530 Subject: [PATCH 3/4] code push issue --- .../ekstep/analytics/exhaust/TestOnDemandDruidExhaustJob.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batch-models/src/test/scala/org/ekstep/analytics/exhaust/TestOnDemandDruidExhaustJob.scala b/batch-models/src/test/scala/org/ekstep/analytics/exhaust/TestOnDemandDruidExhaustJob.scala index bcd7e9f5..35a95525 100644 --- a/batch-models/src/test/scala/org/ekstep/analytics/exhaust/TestOnDemandDruidExhaustJob.scala +++ b/batch-models/src/test/scala/org/ekstep/analytics/exhaust/TestOnDemandDruidExhaustJob.scala @@ -1079,7 +1079,7 @@ object OnDemandDruidExhaustTestJob extends BaseReportsJob with Serializable with } } -// Test object with zipEnabled = false +// Tests object with zipEnabled = false object OnDemandDruidExhaustTestJob2 extends BaseReportsJob with Serializable with IJob with OnDemandBaseExhaustJob with BaseDruidQueryProcessor { implicit override val className: String = "org.sunbird.analytics.exhaust.OnDemandDruidExhaustTestJob2" From 44b503df1ec3f2a7ed14d319f7810e18d6a91844 Mon Sep 17 00:00:00 2001 From: Shakthieshwari Date: Wed, 29 Jun 2022 18:59:00 +0530 Subject: [PATCH 4/4] code push issue --- .../ekstep/analytics/exhaust/TestOnDemandDruidExhaustJob.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batch-models/src/test/scala/org/ekstep/analytics/exhaust/TestOnDemandDruidExhaustJob.scala b/batch-models/src/test/scala/org/ekstep/analytics/exhaust/TestOnDemandDruidExhaustJob.scala index 35a95525..bcd7e9f5 100644 --- a/batch-models/src/test/scala/org/ekstep/analytics/exhaust/TestOnDemandDruidExhaustJob.scala +++ b/batch-models/src/test/scala/org/ekstep/analytics/exhaust/TestOnDemandDruidExhaustJob.scala @@ -1079,7 +1079,7 @@ object OnDemandDruidExhaustTestJob extends BaseReportsJob with Serializable with } } -// Tests object with zipEnabled = false +// Test object with zipEnabled = false object OnDemandDruidExhaustTestJob2 extends BaseReportsJob with Serializable with IJob with OnDemandBaseExhaustJob with BaseDruidQueryProcessor { implicit override val className: String = "org.sunbird.analytics.exhaust.OnDemandDruidExhaustTestJob2"