From 202d90402d06c7cff4510553668a0bf284468bf4 Mon Sep 17 00:00:00 2001 From: Shakthieshwari Date: Fri, 15 Jul 2022 12:42:42 +0530 Subject: [PATCH 1/4] Adding Druid Extension for HLL Support --- ansible/roles/analytics-druid/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/analytics-druid/defaults/main.yml b/ansible/roles/analytics-druid/defaults/main.yml index 1507ccd5ac..8a55bc3f97 100644 --- a/ansible/roles/analytics-druid/defaults/main.yml +++ b/ansible/roles/analytics-druid/defaults/main.yml @@ -33,7 +33,7 @@ druid_request_logging_type: "file" druid_storage_type: "azure" -druid_extensions_list : '"druid-azure-extensions", "graphite-emitter", "postgresql-metadata-storage", "druid-kafka-indexing-service"' +druid_extensions_list : '"druid-azure-extensions", "graphite-emitter", "postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-datasketches"' druid_community_extensions: - graphite-emitter From 67e320819547ae9ad4eeecbb50416db9dc44db26 Mon Sep 17 00:00:00 2001 From: Shakthieshwari Date: Fri, 29 Jul 2022 17:24:28 +0530 Subject: [PATCH 2/4] ML Entity Cols Ingestion Spec Updates --- .../templates/raw_sl_observation | 28 ++++++++++++------- .../templates/raw_sl_observation_status | 10 +++---- .../templates/raw_sl_project_index | 10 +++---- .../druid-ingestion/templates/raw_sl_survey | 11 ++++---- 4 files changed, 33 insertions(+), 26 deletions(-) diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_observation b/ansible/roles/druid-ingestion/templates/raw_sl_observation index 486feb7e19..f2a0a7da72 100644 --- a/ansible/roles/druid-ingestion/templates/raw_sl_observation +++ b/ansible/roles/druid-ingestion/templates/raw_sl_observation @@ -252,14 +252,6 @@ "type": "string", "name": "role_title" }, - { - "type": "string", - "name": "location_validated_with_geotag" - }, - { - "type": "string", - "name": "distance_in_meters" - }, { "type": "string", "name": "entity" @@ -422,11 +414,11 @@ }, { "type": "string", - "name": "user_schoolId" + "name": "school_externalId" }, { "type": "string", - "name": "user_schoolUDISE_code" + "name": "school_code" }, { "type": "string", @@ -463,6 +455,22 @@ { "type": "string", "name": "user_type" + }, + { + "type": "string", + "name": "state_code" + }, + { + "type": "string", + "name": "block_code" + }, + { + "type": "string", + "name": "district_code" + }, + { + "type": "string", + "name": "cluster_code" } ] } diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_observation_status b/ansible/roles/druid-ingestion/templates/raw_sl_observation_status index 72d7e872b5..2d49030ca7 100644 --- a/ansible/roles/druid-ingestion/templates/raw_sl_observation_status +++ b/ansible/roles/druid-ingestion/templates/raw_sl_observation_status @@ -83,7 +83,7 @@ }, { "type": "string", - "name": "school_id" + "name": "school_code" }, { "type": "string", @@ -95,7 +95,7 @@ }, { "type": "string", - "name": "state_id" + "name": "state_code" }, { "type": "string", @@ -107,7 +107,7 @@ }, { "type": "string", - "name": "district_id" + "name": "district_code" }, { "type": "string", @@ -119,7 +119,7 @@ }, { "type": "string", - "name": "block_id" + "name": "block_code" }, { "type": "string", @@ -131,7 +131,7 @@ }, { "type": "string", - "name": "cluster_id" + "name": "cluster_code" }, { "type": "string", diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_project_index b/ansible/roles/druid-ingestion/templates/raw_sl_project_index index 3ef92f6e70..1e6881d72d 100644 --- a/ansible/roles/druid-ingestion/templates/raw_sl_project_index +++ b/ansible/roles/druid-ingestion/templates/raw_sl_project_index @@ -43,7 +43,7 @@ }, { "type": "string", - "name": "school_id" + "name": "school_code" }, { "type": "string", @@ -55,7 +55,7 @@ }, { "type": "string", - "name": "block_id" + "name": "block_code" }, { "type": "string", @@ -67,7 +67,7 @@ }, { "type": "string", - "name": "district_id" + "name": "district_code" }, { "type": "string", @@ -79,7 +79,7 @@ }, { "type": "string", - "name": "cluster_id" + "name": "cluster_code" }, { "type": "string", @@ -91,7 +91,7 @@ }, { "type": "string", - "name": "state_id" + "name": "state_code" }, { "type": "string", diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_survey b/ansible/roles/druid-ingestion/templates/raw_sl_survey index ffb9cdd383..18719f9a7f 100644 --- a/ansible/roles/druid-ingestion/templates/raw_sl_survey +++ b/ansible/roles/druid-ingestion/templates/raw_sl_survey @@ -208,7 +208,6 @@ "type": "string", "name": "organisation_name" }, - , { "type": "string", "name": "user_subtype" @@ -223,7 +222,7 @@ }, { "type": "string", - "name": "district_id" + "name": "district_code" }, { "type": "string", @@ -235,7 +234,7 @@ }, { "type": "string", - "name": "block_id" + "name": "block_code" }, { "type": "string", @@ -247,7 +246,7 @@ }, { "type": "string", - "name": "school_id" + "name": "school_code" }, { "type": "string", @@ -259,7 +258,7 @@ }, { "type": "string", - "name": "cluster_id" + "name": "cluster_code" }, { "type": "string", @@ -271,7 +270,7 @@ }, { "type": "string", - "name": "state_id" + "name": "state_code" }, { "type": "string", From f4d4e74b38ac42c0c68b68920d85a017f3ec771e Mon Sep 17 00:00:00 2001 From: Ashwiniev95 Date: Mon, 29 Aug 2022 19:08:54 +0530 Subject: [PATCH 3/4] Update fields --- .../druid-ingestion/templates/rollup_ml_survey_status | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/roles/druid-ingestion/templates/rollup_ml_survey_status b/ansible/roles/druid-ingestion/templates/rollup_ml_survey_status index 33aec75483..f7b9911af6 100644 --- a/ansible/roles/druid-ingestion/templates/rollup_ml_survey_status +++ b/ansible/roles/druid-ingestion/templates/rollup_ml_survey_status @@ -116,7 +116,7 @@ "type": "jq" }, { - "expr": "if (.programInformation? | .name) then .programInformation? | .name else null end", + "expr": "if (.programInfo? | .name) then .programInfo? | .name else null end", "name": "program_name", "type": "jq" }, @@ -131,7 +131,7 @@ "type": "jq" }, { - "expr": "if (.surveyInformation? | .solutionExternalId) then .surveyInformation? | .solutionExternalId else null end", + "expr": ".solutionExternalId", "name": "survey_externalId", "type": "jq" }, @@ -153,7 +153,7 @@ { "type": "jq", "name": "solution_name", - "expr": "if (.solutionInformation? | .name) then .solutionInformation? | .name else null end" + "expr": "if (.solutionInfo? | .name) then .solutionInfo? | .name else null end" }, { "type": "jq", @@ -191,7 +191,7 @@ { "type": "expression", "name": "status_code", - "expression": "case_simple(\"status\",'started',1,'inprogress',3,'ratingPending',5,'completed',7,0)" + "expression": "case_simple(\"status\",'started',1,'inProgress',3,'ratingPending',5,'completed',7,0)" } ] }, From 780f942035670688192e449857fd8e2a77519824 Mon Sep 17 00:00:00 2001 From: Shakthieshwari Date: Tue, 30 Aug 2022 18:49:42 +0530 Subject: [PATCH 4/4] evidence column addition in survey raw ingestion spec --- ansible/roles/druid-ingestion/templates/raw_sl_survey | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_survey b/ansible/roles/druid-ingestion/templates/raw_sl_survey index 18719f9a7f..85c4b94e05 100644 --- a/ansible/roles/druid-ingestion/templates/raw_sl_survey +++ b/ansible/roles/druid-ingestion/templates/raw_sl_survey @@ -160,6 +160,10 @@ "type": "string", "name": "evidence_count" }, + { + "type": "string", + "name": "evidences" + }, { "type": "string", "name": "instanceParentCriteriaId"