From 634bb0a4ca06f60bf6da96e093eb2ab55493c581 Mon Sep 17 00:00:00 2001 From: VISHNUDAS <95604247+VISHNUDAS-tunerlabs@users.noreply.github.com> Date: Thu, 27 Apr 2023 10:00:34 +0530 Subject: [PATCH 01/53] Devops changes for Program join stories (ED-599) && Block Level Filters (ED-94) (#3737) * Devops changes for PII data stories * env variable change * env variable and kong api name changed: Review fix * ED-94 Devops changes --- ansible/roles/kong-api/defaults/main.yml | 27 +++++++++++++++++-- .../templates/ml-core-service.env | 14 +++++++++- .../templates/ml-reports-service.env | 13 +++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 7af41e2149..2eb03d3a85 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -123,6 +123,8 @@ uci_admin_prefix: /uci notification_service_prefix: /notification registry_service_prefix: /rc assessment_prefix: /assessment +programs_service_prefix: /programs +resource_prefix: /resource # Service URLs knowledge_mw_service_url: "http://knowledge-mw-service:5000" @@ -7522,8 +7524,8 @@ kong_apis: config.enabled: false - name: getSolutionReportInformation - uris: "{{ solutions_service_prefix }}/mlcore/v1/read" - upstream_url: "{{ ml_core_service_url }}/v1/solutions/read" + uris: "{{ resource_prefix }}/mlreports/v1/filtervalues" + upstream_url: "{{{ ml_reports_service_url }}/v1/resource/filtervalues" strip_uri: true plugins: - name: jwt @@ -10001,3 +10003,24 @@ kong_apis: - name: opa-checks config.required: true config.enabled: true + +- name: joinProgramByUser + uris: "{{ programs_service_prefix }}/mlcore/v1/join" + upstream_url: "{{ ml_core_service_url }}/v1/programs/join" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - programAccess + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false diff --git a/ansible/roles/stack-sunbird/templates/ml-core-service.env b/ansible/roles/stack-sunbird/templates/ml-core-service.env index e3db337464..2795c258e1 100755 --- a/ansible/roles/stack-sunbird/templates/ml-core-service.env +++ b/ansible/roles/stack-sunbird/templates/ml-core-service.env @@ -81,4 +81,16 @@ ELASTICSEARCH_ENTITIES_INDEX={{ml_core_elasticsearch_user_extension_index_type | USER_SERVICE_URL={{ml_core_user_service_URL | default("http://learner-service:9000")}} ## portal url of env -APP_PORTAL_BASE_URL={{ proto }}://{{ domain_name }} \ No newline at end of file +APP_PORTAL_BASE_URL={{ proto }}://{{ domain_name }} + +# Kafka enable or disable communication flag +KAFKA_COMMUNICATIONS_ON_OFF={{ml_core_kafka_communications_on_off | default("ON")}} + +# IP address of kafka server with port without HTTP +KAFKA_URL={{ml_core_kafka_url | default(groups['processing-cluster-kafka'][0]+':9092')}} + +# Kafka consumer group for ML Core Service +KAFKA_GROUP_ID={{ml_core_kafka_group_id | default("ml-core-service")}} + +#Kafka topic name for pushing programUsers +PROGRAM_USERS_JOINED_TOPIC={{ml_core_program_users_joined_topic | default (env_name+".programuser.info")}} \ No newline at end of file diff --git a/ansible/roles/stack-sunbird/templates/ml-reports-service.env b/ansible/roles/stack-sunbird/templates/ml-reports-service.env index 6df09e49c6..744a2d124d 100644 --- a/ansible/roles/stack-sunbird/templates/ml-reports-service.env +++ b/ansible/roles/stack-sunbird/templates/ml-reports-service.env @@ -20,6 +20,19 @@ SURVEY_DATASOURCE_NAME={{ml_report_survey_datasource_name | default("sl-survey") # Survey evidence data source name SURVEY_EVIDENCE_DATASOURCE_NAME={{ml_report_survey_evidence_datasource_name | default("sl-survey-evidence")}} +# Project Solution Resource data source name +PROJECT_RESOURCE_DATASOURCE_NAME = {{ml_project_resource_datasource_name | default("ml-project-status")}} + +# Observation Solution Resource data source name +OBSERVATION_RESOURCE_DATASOURCE_NAME = {{ml_observation_resource_datasource_name | default("ml-obs-status")}} + +# Survey Solution Resource data source name +SURVEY_RESOURCE_DATASOURCE_NAME = {{ml_survey_resource_datasource_name | default("ml-survey-status")}} + + +# Program Resource data source name +PROGRAM_RESOURCE_DATASOURCE_NAME = {{ml_program_resource_datasource_name | default("ml-user-program")}} + # Dhiti Thresold Values #Restrict number of records to be shown for container reports CONTENT_REPORT_THRESHOLD={{ml_report_content_report_threshold | default("5")}} From b626c0008998d2db6dbd07dd5e6ebb7b6b132029 Mon Sep 17 00:00:00 2001 From: Akash Shah Date: Wed, 3 May 2023 16:35:49 +0530 Subject: [PATCH 02/53] Fix syntax for API getSolutionReportInformation --- ansible/roles/kong-api/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 2eb03d3a85..4c217d1692 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -7525,7 +7525,7 @@ kong_apis: - name: getSolutionReportInformation uris: "{{ resource_prefix }}/mlreports/v1/filtervalues" - upstream_url: "{{{ ml_reports_service_url }}/v1/resource/filtervalues" + upstream_url: "{{ ml_reports_service_url }}/v1/resource/filtervalues" strip_uri: true plugins: - name: jwt From e049397be32b7e094c5ac7c2a08cc781535a1edf Mon Sep 17 00:00:00 2001 From: VISHNUDAS-tunerlabse Date: Thu, 11 May 2023 14:10:03 +0530 Subject: [PATCH 03/53] ml-reports env syntax error fix added --- .../roles/stack-sunbird/templates/ml-reports-service.env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/roles/stack-sunbird/templates/ml-reports-service.env b/ansible/roles/stack-sunbird/templates/ml-reports-service.env index 744a2d124d..cd6754ac01 100644 --- a/ansible/roles/stack-sunbird/templates/ml-reports-service.env +++ b/ansible/roles/stack-sunbird/templates/ml-reports-service.env @@ -21,17 +21,17 @@ SURVEY_DATASOURCE_NAME={{ml_report_survey_datasource_name | default("sl-survey") SURVEY_EVIDENCE_DATASOURCE_NAME={{ml_report_survey_evidence_datasource_name | default("sl-survey-evidence")}} # Project Solution Resource data source name -PROJECT_RESOURCE_DATASOURCE_NAME = {{ml_project_resource_datasource_name | default("ml-project-status")}} +PROJECT_RESOURCE_DATASOURCE_NAME={{ml_project_resource_datasource_name | default("ml-project-status")}} # Observation Solution Resource data source name -OBSERVATION_RESOURCE_DATASOURCE_NAME = {{ml_observation_resource_datasource_name | default("ml-obs-status")}} +OBSERVATION_RESOURCE_DATASOURCE_NAME={{ml_observation_resource_datasource_name | default("ml-obs-status")}} # Survey Solution Resource data source name -SURVEY_RESOURCE_DATASOURCE_NAME = {{ml_survey_resource_datasource_name | default("ml-survey-status")}} +SURVEY_RESOURCE_DATASOURCE_NAME={{ml_survey_resource_datasource_name | default("ml-survey-status")}} # Program Resource data source name -PROGRAM_RESOURCE_DATASOURCE_NAME = {{ml_program_resource_datasource_name | default("ml-user-program")}} +PROGRAM_RESOURCE_DATASOURCE_NAME={{ml_program_resource_datasource_name | default("ml-user-program")}} # Dhiti Thresold Values #Restrict number of records to be shown for container reports From 2f67f7a43daa4cdc0ccc86eb2d7b5b35ac5c76d4 Mon Sep 17 00:00:00 2001 From: princegupta1131 <114015020+princegupta1131@users.noreply.github.com> Date: Mon, 15 May 2023 11:56:35 +0530 Subject: [PATCH 04/53] ED-1945 feat:onboarded Question-set,question copy api --- ansible/roles/kong-api/defaults/main.yml | 42 ++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 4c217d1692..c59e52bf17 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -7459,6 +7459,48 @@ kong_apis: - name: opa-checks config.required: false config.enabled: false + +- name: questionsetCopy + uris: "{{ questionset_prefix }}/v1/copy" + upstream_url: "{{ assessment_service_url }}/questionset/v4/copy" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionCopy + uris: "{{ question_prefix }}/v1/copy" + upstream_url: "{{ assessment_service_url }}/question/v4/copy" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false - name: getUsersPrograms uris: "{{ users_service_prefix }}/mlcore/v1/programs" From 9cdd587a2e7632df2e40b96cdd12b5c5f2028f13 Mon Sep 17 00:00:00 2001 From: VISHNUDAS-tunerlabse Date: Mon, 15 May 2023 16:58:33 +0530 Subject: [PATCH 05/53] programuser.info Kafka topic added to script --- ansible/roles/setup-kafka/defaults/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ansible/roles/setup-kafka/defaults/main.yml b/ansible/roles/setup-kafka/defaults/main.yml index cb2bb2f38f..293a9f9988 100644 --- a/ansible/roles/setup-kafka/defaults/main.yml +++ b/ansible/roles/setup-kafka/defaults/main.yml @@ -83,6 +83,10 @@ ml_service_topics: retention_time: 172800000 replication_factor: 1 num_of_partitions: 1 + - name: programuser.info + retention_time: 172800000 + replication_factor: 1 + num_of_partitions: 1 uci_service_topics: - name: inbound-error From 3e657821d09a4e5e3abe94f0e6fb8ec271e7cf55 Mon Sep 17 00:00:00 2001 From: princegupta1131 <114015020+princegupta1131@users.noreply.github.com> Date: Thu, 18 May 2023 13:44:34 +0530 Subject: [PATCH 06/53] ED-1945 feat:onboarded Question-set,question copy api (#3820) --- ansible/roles/kong-api/defaults/main.yml | 42 ++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 4c217d1692..c59e52bf17 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -7459,6 +7459,48 @@ kong_apis: - name: opa-checks config.required: false config.enabled: false + +- name: questionsetCopy + uris: "{{ questionset_prefix }}/v1/copy" + upstream_url: "{{ assessment_service_url }}/questionset/v4/copy" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionCopy + uris: "{{ question_prefix }}/v1/copy" + upstream_url: "{{ assessment_service_url }}/question/v4/copy" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false - name: getUsersPrograms uris: "{{ users_service_prefix }}/mlcore/v1/programs" From 2998eec2c2d4d63c09434036bd074127bc1fc234 Mon Sep 17 00:00:00 2001 From: sachin patil <91308046+spsac08@users.noreply.github.com> Date: Thu, 18 May 2023 14:31:23 +0530 Subject: [PATCH 07/53] Update main.yml Updated the value and added new keys and Values --- ansible/roles/ml-analytics-service/defaults/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index 15f9b438c4..c859d0da48 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -58,7 +58,7 @@ ml_analytics_druid_project_injestion_spec : '{"type":"index","spec":{"ioConfig": ml_analytics_druid_distinctCnt_obs_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount/ml_observation_distinctCount_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' ml_analytics_druid_distinctCnt_obs_domain_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain/ml_observation_distinctCount_domain.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain_criteria/ml_observation_distinctCount_domain_criteria.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain-criteria","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"string","name":"criteria_name"},{"type":"string","name":"criteria_score"},{"type":"string","name":"criteria_id"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' -ml_analytics_druid_distinctCnt_projects_status_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/distinctCount/ml_projects_distinctCount.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"project_title"},{"type":"string","name":"solution_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"},{"type":"long","name":"no_of_certificate_issued"}]},"metricsSpec":[]}}}' +ml_analytics_druid_distinctCnt_projects_status_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/distinctCount/ml_projects_distinctCount.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"project_title"},{"type":"string","name":"solution_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"long","name":"no_of_imp_with_evidence_inprogress"},{"type":"string","name":"time_stamp"},{"type":"long","name":"no_of_certificate_issued"}]},"metricsSpec":[]}}}' ml_analytics_obs_distinctCnt_cloud_blob_path: "observation/distinctCount/" ml_analytics_obs_distinctCnt_domain_cloud_blob_path: "observation/distinctCount_domain/" ml_analytics_obs_distinctCnt_domain_criteria_cloud_blob_path: "observation/distinctCount_domain_criteria/" @@ -116,3 +116,5 @@ ml_analytics_project_program : "{{ WORKDIR }}/ml-analytics-service/projects/prog ml_analytics_projects_program_filename: "{{ config_path }}/projects/program_ids.txt" ml_analytics_nvsk_imp_projects_data_local_path: "{{ config_path }}/urgent_data_metrics/output/" ml_analytics_nvsk_imp_projects_data_blob_path: "Manage_Learn_Data/micro_improvement/" +ml_analytics_nvsk_imp_projects_state_data_local_path: "{{ config_path }}/urgent_data_metrics/output_state/" +ml_analytics_nvsk_imp_projects_state_data_blob_path: "Manage_Learn_Data/micro_improvement_state/" From c5687dc87159cc050de774baf25188f72f71861c Mon Sep 17 00:00:00 2001 From: princegupta1131 <114015020+princegupta1131@users.noreply.github.com> Date: Tue, 6 Jun 2023 12:41:10 +0530 Subject: [PATCH 08/53] #SB-0000 fix: updating the pdataid and version --- .../artifacts/sunbird/login/resources/js/telemetry_service.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js b/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js index cd664d20eb..e858bfb817 100644 --- a/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js +++ b/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js @@ -2488,6 +2488,8 @@ if (window.location.origin.indexOf("diksha.gov.in") >= 0) { tenantSlug = "sunbird"; } else { pdataId = "preprod.diksha.portal"; + tenantSlug = "sunbird"; + } let curUrlObj = window.location; @@ -2506,7 +2508,7 @@ if(client_id.toLowerCase() === 'android'){ "telemetry": { "pdata": { "id": pdataId, - "ver": "5.2.0", + "ver": "6.0.0", "pid": "sunbird-portal" } } From 5d52491523e023d329276958e93ec776b601ef2c Mon Sep 17 00:00:00 2001 From: Shakthieshwari Date: Tue, 6 Jun 2023 17:00:58 +0530 Subject: [PATCH 09/53] Added ML PII Data Product job to submit APO --- kubernetes/opa/analytics/policies.rego | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/opa/analytics/policies.rego b/kubernetes/opa/analytics/policies.rego index a76d5d03ac..b590c6e3e8 100644 --- a/kubernetes/opa/analytics/policies.rego +++ b/kubernetes/opa/analytics/policies.rego @@ -57,7 +57,7 @@ submitDataExhaustRequest { roles :=["PROGRAM_MANAGER", "PROGRAM_DESIGNER"] super.acls_check(acls) super.role_check(roles) - input.parsed_body.request.dataset in ["druid-dataset"] + input.parsed_body.request.dataset in ["druid-dataset","program-user-exhaust"] x_channel_id x_authenticated_userid == super.userid } @@ -65,4 +65,4 @@ submitDataExhaustRequest { submitDataExhaustRequest { not x_authenticated_user_token not x_authenticated_userid -} \ No newline at end of file +} From 217e1d2ae7943e7926c9885723d83b2878a353d5 Mon Sep 17 00:00:00 2001 From: Ashwiniev95 Date: Mon, 19 Jun 2023 17:37:42 +0530 Subject: [PATCH 10/53] Add variable forreport automation --- .../ml-analytics-service/defaults/main.yml | 27 ++++++++ .../ml-analytics-service/templates/config.j2 | 64 +++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index c859d0da48..006dab3da6 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -118,3 +118,30 @@ ml_analytics_nvsk_imp_projects_data_local_path: "{{ config_path }}/urgent_data_m ml_analytics_nvsk_imp_projects_data_blob_path: "Manage_Learn_Data/micro_improvement/" ml_analytics_nvsk_imp_projects_state_data_local_path: "{{ config_path }}/urgent_data_metrics/output_state/" ml_analytics_nvsk_imp_projects_state_data_blob_path: "Manage_Learn_Data/micro_improvement_state/" +ml_analytics_authorization_access_token : "{{ ml_analytics_authorization_access_token }}" +ml_analytics_content_type_url : "{{ ml_analytics_content_type_url }}" +ml_analytics_mongo_reports_log_collec : "{{ ml_analytics_mongo_reports_log_collec }}" +ml_analytics_client_id : "{{ ml_analytics_client_id }}" +ml_analytics_client_secret : "{{ ml_analytics_client_secret }}" +ml_analytics_grant_type : "{{ ml_analytics_grant_type }}" +ml_analytics_username : "{{ ml_analytics_username }}" +ml_analytics_password : "{{ ml_analytics_password }}" +ml_analytics_api_base_url : "{{ ml_analytics_api_base_url }}" +ml_analytics_refresh_token : "auth/realms/sunbird/protocol/openid-connect/token" +ml_analytics_access_token : "auth/v1/refresh/token" +ml_analytics_backend_create : "api/data/v1/report/jobs/submit" +ml_analytics_frontend_create : "/api/data/v1/report-service/report/create" +ml_analytics_frontend_get : "/api/data/v1/report-service/report/get/" +ml_analytics_frontend_update : "/api/data/v1/report-service/report/update/" +ml_analytics_backend_update : "/api/data/v1/report-service/report/update/" +ml_analytics_frontend_retire : "/api/data/v1/report-service/report/delete/" +ml_analytics_backend_retire : "/report/jobs/deactivate/" +ml_analytics_folder_name : 6.0.0 +ml_analytics_script_path : "/opt/sparkjobs/ml-analytics-service/migrations/lib" +ml_analytics_folder_config : "/opt/sparkjobs/ml-analytics-service/migrations/releases/6.0.0/config/" +ml_analytics_createdBy : "{{ ml_analytics_createdBy }}" +ml_analytics_reports_container : "{{ ml_analytics_reports_container }}" +ml_analytics_reports_store : "{{ ml_analytics_reports_store }}" +ml_analytics_reports_key : "{{ ml_analytics_reports_key }}" +ml_analytics_update_tag : "{{ ml_analytics_update_tag }}" + diff --git a/ansible/roles/ml-analytics-service/templates/config.j2 b/ansible/roles/ml-analytics-service/templates/config.j2 index 70fe0ff018..f0ecf38bc8 100644 --- a/ansible/roles/ml-analytics-service/templates/config.j2 +++ b/ansible/roles/ml-analytics-service/templates/config.j2 @@ -15,6 +15,10 @@ authorization = {{ ml_analytics_api_authorization_key }} internal_access_token = {{ ml_analytics_api_access_token }} +authorization_access_token = {{ ml_analytics_authorization_access_token }} + +content_type_url = {{ ml_analytics_content_type_url }} + [ML_SURVEY_SERVICE_URL] @@ -61,6 +65,8 @@ survey_submissions_collection = {{ ml_analytics_mongo_survey_submissions_collect survey_collection = {{ ml_analytics_mongo_survey_collection }} +reports_log_collec = {{ ml_analytics_mongo_reports_log_collec }} + [DRUID] @@ -283,3 +289,61 @@ cname_url = {{ ml_analytics_cname_url }} nvsk_imp_projects_data_local_path = {{ ml_analytics_nvsk_imp_projects_data_local_path }} nvsk_imp_projects_data_blob_path = {{ ml_analytics_nvsk_imp_projects_data_blob_path }} + +[API_CREDENTIALS] + +client_id = {{ ml_analytics_client_id }} + +client_secret = {{ ml_analytics_client_secret }} + +grant_type = {{ ml_analytics_grant_type }} + +username = {{ ml_analytics_username }} + +password = {{ ml_analytics_password }} + +[API_ENDPOINTS] + +base_url = {{ ml_analytics_api_base_url }} + +refresh_token = {{ ml_analytics_refresh_token }} + +access_token = {{ ml_analytics_access_token }} + +backend_create = {{ ml_analytics_backend_create }} + +frontend_create = {{ ml_analytics_frontend_create }} + +frontend_get = {{ ml_analytics_frontend_get }} + +frontend_update = {{ ml_analytics_frontend_update }} + +backend_update = {{ ml_analytics_backend_update }} + +frontend_retire = {{ ml_analytics_frontend_retire }} + +backend_retire = {{ ml_analytics_backend_retire }} + + +[REPORTS_FILEPATH] + +folder_name = {{ ml_analytics_folder_name }} + +script_path = {{ ml_analytics_script_path }} + +folder_config = {{ ml_analytics_folder_config }} + + +[JSON_VARIABLE] + +createdBy = {{ ml_analytics_createdBy }} + +container = {{ ml_analytics_reports_container }} + +store = {{ ml_analytics_reports_store }} + +key = {{ ml_analytics_reports_key }} + +[REPORT_IDS] + +update_tag = {{ ml_analytics_update_tag }} From f5c211d202aa1654bde3620c5ef0152bbf0d72b2 Mon Sep 17 00:00:00 2001 From: Abhishek Nagesh Date: Tue, 20 Jun 2023 10:07:32 +0530 Subject: [PATCH 11/53] Issue #ED-0000 fix:added env variable to store default board value --- ansible/roles/stack-sunbird/templates/sunbird_player.env | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/roles/stack-sunbird/templates/sunbird_player.env b/ansible/roles/stack-sunbird/templates/sunbird_player.env index 30d4e1830f..0f749f913a 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_player.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_player.env @@ -233,3 +233,6 @@ cloud_private_storage_project={{cloud_private_storage_project | default("")}} cloud_storage_privatereports_bucketname={{cloud_storage_privatereports_bucketname | default("reports")}} cloud_storage_resourceBundle_bucketname={{cloud_storage_label_bucketname | default("label")}} cloud_storage_desktopCrash_bucketname={{cloud_storage_desktopcrash_bucketname | default("desktopappcrashlogs")}} + +#release-6.0.0 +sunbird_default_board={{sunbird_default_board}} \ No newline at end of file From 5266249fa623110fba7628026ace10d6f36ccfbb Mon Sep 17 00:00:00 2001 From: Abhishek Nagesh Date: Tue, 20 Jun 2023 10:20:22 +0530 Subject: [PATCH 12/53] Issue #ED-0000 fix:added fallback for default board --- ansible/roles/stack-sunbird/templates/sunbird_player.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/stack-sunbird/templates/sunbird_player.env b/ansible/roles/stack-sunbird/templates/sunbird_player.env index 0f749f913a..6c2c86f19f 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_player.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_player.env @@ -235,4 +235,4 @@ cloud_storage_resourceBundle_bucketname={{cloud_storage_label_bucketname | defau cloud_storage_desktopCrash_bucketname={{cloud_storage_desktopcrash_bucketname | default("desktopappcrashlogs")}} #release-6.0.0 -sunbird_default_board={{sunbird_default_board}} \ No newline at end of file +sunbird_default_board={{sunbird_default_board | default("")}} \ No newline at end of file From ff5b2d351b8bc1e96642bcb9659358eca81d8cbd Mon Sep 17 00:00:00 2001 From: ankitshahu Date: Tue, 20 Jun 2023 04:52:48 +0000 Subject: [PATCH 13/53] Added env key for timezone Difference --- ansible/roles/stack-sunbird/templates/ml-core-service.env | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ansible/roles/stack-sunbird/templates/ml-core-service.env b/ansible/roles/stack-sunbird/templates/ml-core-service.env index 2795c258e1..3d2277b8f8 100755 --- a/ansible/roles/stack-sunbird/templates/ml-core-service.env +++ b/ansible/roles/stack-sunbird/templates/ml-core-service.env @@ -93,4 +93,7 @@ KAFKA_URL={{ml_core_kafka_url | default(groups['processing-cluster-kafka'][0]+': KAFKA_GROUP_ID={{ml_core_kafka_group_id | default("ml-core-service")}} #Kafka topic name for pushing programUsers -PROGRAM_USERS_JOINED_TOPIC={{ml_core_program_users_joined_topic | default (env_name+".programuser.info")}} \ No newline at end of file +PROGRAM_USERS_JOINED_TOPIC={{ml_core_program_users_joined_topic | default (env_name+".programuser.info")}} + +#TimeOffset with respect to UTC +TIMEZONE_DIFFRENECE_BETWEEN_LOCAL_TIME_AND_UTC={{ml_core_timezone_diffrenece_between_local_time_and_utc | default ("+05:30))}} \ No newline at end of file From aba7e5f71fcc55529b265587ce4d98e27c7bb8a3 Mon Sep 17 00:00:00 2001 From: ankitshahu Date: Tue, 20 Jun 2023 04:53:55 +0000 Subject: [PATCH 14/53] Added env key for timezone Difference --- ansible/roles/stack-sunbird/templates/ml-core-service.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/stack-sunbird/templates/ml-core-service.env b/ansible/roles/stack-sunbird/templates/ml-core-service.env index 3d2277b8f8..0f9f0e4963 100755 --- a/ansible/roles/stack-sunbird/templates/ml-core-service.env +++ b/ansible/roles/stack-sunbird/templates/ml-core-service.env @@ -96,4 +96,4 @@ KAFKA_GROUP_ID={{ml_core_kafka_group_id | default("ml-core-service")}} PROGRAM_USERS_JOINED_TOPIC={{ml_core_program_users_joined_topic | default (env_name+".programuser.info")}} #TimeOffset with respect to UTC -TIMEZONE_DIFFRENECE_BETWEEN_LOCAL_TIME_AND_UTC={{ml_core_timezone_diffrenece_between_local_time_and_utc | default ("+05:30))}} \ No newline at end of file +TIMEZONE_DIFFRENECE_BETWEEN_LOCAL_TIME_AND_UTC={{ml_core_timezone_diffrenece_between_local_time_and_utc | default ("+05:30"))}} \ No newline at end of file From cfe06e65de39ae7ce32c1edafcf4348a022113b1 Mon Sep 17 00:00:00 2001 From: ankitshahu Date: Tue, 20 Jun 2023 05:13:24 +0000 Subject: [PATCH 15/53] Added env key for timezone Difference --- ansible/roles/stack-sunbird/templates/ml-core-service.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/stack-sunbird/templates/ml-core-service.env b/ansible/roles/stack-sunbird/templates/ml-core-service.env index 0f9f0e4963..0209e84ae6 100755 --- a/ansible/roles/stack-sunbird/templates/ml-core-service.env +++ b/ansible/roles/stack-sunbird/templates/ml-core-service.env @@ -96,4 +96,4 @@ KAFKA_GROUP_ID={{ml_core_kafka_group_id | default("ml-core-service")}} PROGRAM_USERS_JOINED_TOPIC={{ml_core_program_users_joined_topic | default (env_name+".programuser.info")}} #TimeOffset with respect to UTC -TIMEZONE_DIFFRENECE_BETWEEN_LOCAL_TIME_AND_UTC={{ml_core_timezone_diffrenece_between_local_time_and_utc | default ("+05:30"))}} \ No newline at end of file +TIMEZONE_DIFFRENECE_BETWEEN_LOCAL_TIME_AND_UTC={{ml_core_timezone_diffrenece_between_local_time_and_utc | default ("+05:30")}} \ No newline at end of file From 696ab858a1a71e45ed8929c48a5b1232cf6cf0d3 Mon Sep 17 00:00:00 2001 From: Ashwiniev95 Date: Tue, 20 Jun 2023 13:53:35 +0530 Subject: [PATCH 16/53] update variables --- .../ml-analytics-service/defaults/main.yml | 14 ++------- .../ml-analytics-service/templates/config.j2 | 31 ------------------- .../templates/shell_script_config.j2 | 2 ++ 3 files changed, 4 insertions(+), 43 deletions(-) diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index 006dab3da6..8ccb540082 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -119,7 +119,6 @@ ml_analytics_nvsk_imp_projects_data_blob_path: "Manage_Learn_Data/micro_improvem ml_analytics_nvsk_imp_projects_state_data_local_path: "{{ config_path }}/urgent_data_metrics/output_state/" ml_analytics_nvsk_imp_projects_state_data_blob_path: "Manage_Learn_Data/micro_improvement_state/" ml_analytics_authorization_access_token : "{{ ml_analytics_authorization_access_token }}" -ml_analytics_content_type_url : "{{ ml_analytics_content_type_url }}" ml_analytics_mongo_reports_log_collec : "{{ ml_analytics_mongo_reports_log_collec }}" ml_analytics_client_id : "{{ ml_analytics_client_id }}" ml_analytics_client_secret : "{{ ml_analytics_client_secret }}" @@ -127,21 +126,12 @@ ml_analytics_grant_type : "{{ ml_analytics_grant_type }}" ml_analytics_username : "{{ ml_analytics_username }}" ml_analytics_password : "{{ ml_analytics_password }}" ml_analytics_api_base_url : "{{ ml_analytics_api_base_url }}" -ml_analytics_refresh_token : "auth/realms/sunbird/protocol/openid-connect/token" -ml_analytics_access_token : "auth/v1/refresh/token" -ml_analytics_backend_create : "api/data/v1/report/jobs/submit" -ml_analytics_frontend_create : "/api/data/v1/report-service/report/create" -ml_analytics_frontend_get : "/api/data/v1/report-service/report/get/" -ml_analytics_frontend_update : "/api/data/v1/report-service/report/update/" -ml_analytics_backend_update : "/api/data/v1/report-service/report/update/" -ml_analytics_frontend_retire : "/api/data/v1/report-service/report/delete/" -ml_analytics_backend_retire : "/report/jobs/deactivate/" -ml_analytics_folder_name : 6.0.0 ml_analytics_script_path : "/opt/sparkjobs/ml-analytics-service/migrations/lib" -ml_analytics_folder_config : "/opt/sparkjobs/ml-analytics-service/migrations/releases/6.0.0/config/" ml_analytics_createdBy : "{{ ml_analytics_createdBy }}" ml_analytics_reports_container : "{{ ml_analytics_reports_container }}" ml_analytics_reports_store : "{{ ml_analytics_reports_store }}" ml_analytics_reports_key : "{{ ml_analytics_reports_key }}" ml_analytics_update_tag : "{{ ml_analytics_update_tag }}" +ml_analytics_driver_memory: "{{ ml_analytics_driver_memory | default('50g') }}" +ml_analytics_executor_memory: "{{ ml_analytics_executor_memory | default('50g') }}" diff --git a/ansible/roles/ml-analytics-service/templates/config.j2 b/ansible/roles/ml-analytics-service/templates/config.j2 index f0ecf38bc8..1789286872 100644 --- a/ansible/roles/ml-analytics-service/templates/config.j2 +++ b/ansible/roles/ml-analytics-service/templates/config.j2 @@ -17,9 +17,6 @@ internal_access_token = {{ ml_analytics_api_access_token }} authorization_access_token = {{ ml_analytics_authorization_access_token }} -content_type_url = {{ ml_analytics_content_type_url }} - - [ML_SURVEY_SERVICE_URL] url = {{ ml_analytics_survey_service }} @@ -302,38 +299,10 @@ username = {{ ml_analytics_username }} password = {{ ml_analytics_password }} -[API_ENDPOINTS] - -base_url = {{ ml_analytics_api_base_url }} - -refresh_token = {{ ml_analytics_refresh_token }} - -access_token = {{ ml_analytics_access_token }} - -backend_create = {{ ml_analytics_backend_create }} - -frontend_create = {{ ml_analytics_frontend_create }} - -frontend_get = {{ ml_analytics_frontend_get }} - -frontend_update = {{ ml_analytics_frontend_update }} - -backend_update = {{ ml_analytics_backend_update }} - -frontend_retire = {{ ml_analytics_frontend_retire }} - -backend_retire = {{ ml_analytics_backend_retire }} - - [REPORTS_FILEPATH] -folder_name = {{ ml_analytics_folder_name }} - script_path = {{ ml_analytics_script_path }} -folder_config = {{ ml_analytics_folder_config }} - - [JSON_VARIABLE] createdBy = {{ ml_analytics_createdBy }} diff --git a/ansible/roles/ml-analytics-service/templates/shell_script_config.j2 b/ansible/roles/ml-analytics-service/templates/shell_script_config.j2 index 97e35a4db0..e04fbd4418 100644 --- a/ansible/roles/ml-analytics-service/templates/shell_script_config.j2 +++ b/ansible/roles/ml-analytics-service/templates/shell_script_config.j2 @@ -1 +1,3 @@ projects_program_filename={{ ml_analytics_projects_program_filename }} +driver_memory={{ml_analytics_driver_memory}} +executor_memory={{ml_analytics_executor_memory}} From c3e0f45d8f537ce0e26f0188d2fb320987581c24 Mon Sep 17 00:00:00 2001 From: Ashwiniev95 Date: Tue, 20 Jun 2023 16:49:13 +0530 Subject: [PATCH 17/53] remove a variable --- ansible/roles/ml-analytics-service/defaults/main.yml | 1 - ansible/roles/ml-analytics-service/templates/config.j2 | 2 -- 2 files changed, 3 deletions(-) diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index 8ccb540082..92721e0104 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -130,7 +130,6 @@ ml_analytics_script_path : "/opt/sparkjobs/ml-analytics-service/migrations/lib" ml_analytics_createdBy : "{{ ml_analytics_createdBy }}" ml_analytics_reports_container : "{{ ml_analytics_reports_container }}" ml_analytics_reports_store : "{{ ml_analytics_reports_store }}" -ml_analytics_reports_key : "{{ ml_analytics_reports_key }}" ml_analytics_update_tag : "{{ ml_analytics_update_tag }}" ml_analytics_driver_memory: "{{ ml_analytics_driver_memory | default('50g') }}" ml_analytics_executor_memory: "{{ ml_analytics_executor_memory | default('50g') }}" diff --git a/ansible/roles/ml-analytics-service/templates/config.j2 b/ansible/roles/ml-analytics-service/templates/config.j2 index 1789286872..166c833e55 100644 --- a/ansible/roles/ml-analytics-service/templates/config.j2 +++ b/ansible/roles/ml-analytics-service/templates/config.j2 @@ -311,8 +311,6 @@ container = {{ ml_analytics_reports_container }} store = {{ ml_analytics_reports_store }} -key = {{ ml_analytics_reports_key }} - [REPORT_IDS] update_tag = {{ ml_analytics_update_tag }} From 5f3a35f3c34e8ec365a6e9ba3754a746756a18e7 Mon Sep 17 00:00:00 2001 From: Ashwiniev95 Date: Tue, 20 Jun 2023 18:47:23 +0530 Subject: [PATCH 18/53] remove the variable --- ansible/roles/ml-analytics-service/defaults/main.yml | 2 -- ansible/roles/ml-analytics-service/templates/config.j2 | 6 ------ 2 files changed, 8 deletions(-) diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index 92721e0104..7bc5e8c541 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -122,11 +122,9 @@ ml_analytics_authorization_access_token : "{{ ml_analytics_authorization_access_ ml_analytics_mongo_reports_log_collec : "{{ ml_analytics_mongo_reports_log_collec }}" ml_analytics_client_id : "{{ ml_analytics_client_id }}" ml_analytics_client_secret : "{{ ml_analytics_client_secret }}" -ml_analytics_grant_type : "{{ ml_analytics_grant_type }}" ml_analytics_username : "{{ ml_analytics_username }}" ml_analytics_password : "{{ ml_analytics_password }}" ml_analytics_api_base_url : "{{ ml_analytics_api_base_url }}" -ml_analytics_script_path : "/opt/sparkjobs/ml-analytics-service/migrations/lib" ml_analytics_createdBy : "{{ ml_analytics_createdBy }}" ml_analytics_reports_container : "{{ ml_analytics_reports_container }}" ml_analytics_reports_store : "{{ ml_analytics_reports_store }}" diff --git a/ansible/roles/ml-analytics-service/templates/config.j2 b/ansible/roles/ml-analytics-service/templates/config.j2 index 166c833e55..2b85b70359 100644 --- a/ansible/roles/ml-analytics-service/templates/config.j2 +++ b/ansible/roles/ml-analytics-service/templates/config.j2 @@ -293,16 +293,10 @@ client_id = {{ ml_analytics_client_id }} client_secret = {{ ml_analytics_client_secret }} -grant_type = {{ ml_analytics_grant_type }} - username = {{ ml_analytics_username }} password = {{ ml_analytics_password }} -[REPORTS_FILEPATH] - -script_path = {{ ml_analytics_script_path }} - [JSON_VARIABLE] createdBy = {{ ml_analytics_createdBy }} From 2b913e07bc508f21db4e76ff8fe5345dfa2401a5 Mon Sep 17 00:00:00 2001 From: Ashwiniev95 Date: Wed, 21 Jun 2023 11:38:38 +0530 Subject: [PATCH 19/53] Remove a variable --- ansible/roles/ml-analytics-service/defaults/main.yml | 1 - ansible/roles/ml-analytics-service/templates/config.j2 | 3 --- 2 files changed, 4 deletions(-) diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index 7bc5e8c541..dea0395a56 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -128,7 +128,6 @@ ml_analytics_api_base_url : "{{ ml_analytics_api_base_url }}" ml_analytics_createdBy : "{{ ml_analytics_createdBy }}" ml_analytics_reports_container : "{{ ml_analytics_reports_container }}" ml_analytics_reports_store : "{{ ml_analytics_reports_store }}" -ml_analytics_update_tag : "{{ ml_analytics_update_tag }}" ml_analytics_driver_memory: "{{ ml_analytics_driver_memory | default('50g') }}" ml_analytics_executor_memory: "{{ ml_analytics_executor_memory | default('50g') }}" diff --git a/ansible/roles/ml-analytics-service/templates/config.j2 b/ansible/roles/ml-analytics-service/templates/config.j2 index 2b85b70359..4c480341af 100644 --- a/ansible/roles/ml-analytics-service/templates/config.j2 +++ b/ansible/roles/ml-analytics-service/templates/config.j2 @@ -305,6 +305,3 @@ container = {{ ml_analytics_reports_container }} store = {{ ml_analytics_reports_store }} -[REPORT_IDS] - -update_tag = {{ ml_analytics_update_tag }} From 42b2bd7ea670570a3dec1a3a4dd3f16df8ff9d1f Mon Sep 17 00:00:00 2001 From: Ashwiniev95 Date: Thu, 22 Jun 2023 12:34:38 +0530 Subject: [PATCH 20/53] Remove a variable --- ansible/roles/ml-analytics-service/defaults/main.yml | 2 -- ansible/roles/ml-analytics-service/templates/config.j2 | 4 ---- 2 files changed, 6 deletions(-) diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index dea0395a56..95a5267c1d 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -119,7 +119,6 @@ ml_analytics_nvsk_imp_projects_data_blob_path: "Manage_Learn_Data/micro_improvem ml_analytics_nvsk_imp_projects_state_data_local_path: "{{ config_path }}/urgent_data_metrics/output_state/" ml_analytics_nvsk_imp_projects_state_data_blob_path: "Manage_Learn_Data/micro_improvement_state/" ml_analytics_authorization_access_token : "{{ ml_analytics_authorization_access_token }}" -ml_analytics_mongo_reports_log_collec : "{{ ml_analytics_mongo_reports_log_collec }}" ml_analytics_client_id : "{{ ml_analytics_client_id }}" ml_analytics_client_secret : "{{ ml_analytics_client_secret }}" ml_analytics_username : "{{ ml_analytics_username }}" @@ -130,4 +129,3 @@ ml_analytics_reports_container : "{{ ml_analytics_reports_container }}" ml_analytics_reports_store : "{{ ml_analytics_reports_store }}" ml_analytics_driver_memory: "{{ ml_analytics_driver_memory | default('50g') }}" ml_analytics_executor_memory: "{{ ml_analytics_executor_memory | default('50g') }}" - diff --git a/ansible/roles/ml-analytics-service/templates/config.j2 b/ansible/roles/ml-analytics-service/templates/config.j2 index 4c480341af..3d64637aa5 100644 --- a/ansible/roles/ml-analytics-service/templates/config.j2 +++ b/ansible/roles/ml-analytics-service/templates/config.j2 @@ -62,9 +62,6 @@ survey_submissions_collection = {{ ml_analytics_mongo_survey_submissions_collect survey_collection = {{ ml_analytics_mongo_survey_collection }} -reports_log_collec = {{ ml_analytics_mongo_reports_log_collec }} - - [DRUID] metadata_url = http://{{ ml_analytics_druid_url }}/druid/coordinator/v1/datasources/ @@ -304,4 +301,3 @@ createdBy = {{ ml_analytics_createdBy }} container = {{ ml_analytics_reports_container }} store = {{ ml_analytics_reports_store }} - From f99495cb0c7327ee6f53ca1bace241a93567927c Mon Sep 17 00:00:00 2001 From: adithya_dinesh Date: Thu, 22 Jun 2023 18:16:18 +0530 Subject: [PATCH 21/53] Added base url to config --- ansible/roles/ml-analytics-service/templates/config.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ansible/roles/ml-analytics-service/templates/config.j2 b/ansible/roles/ml-analytics-service/templates/config.j2 index 3d64637aa5..ad762488c6 100644 --- a/ansible/roles/ml-analytics-service/templates/config.j2 +++ b/ansible/roles/ml-analytics-service/templates/config.j2 @@ -7,6 +7,11 @@ integrated_app = {{ ml_analytics_integrated_app_name }} integrated_portal = {{ ml_analytics_integrated_portal }} +[API_ENDPOINTS] + +base_url = {{ ml_analytics_api_base_url }} + + [API_HEADERS] content_type = application/json From d9ffaccb32c8c9231fcd9f36d7d29f1934154374 Mon Sep 17 00:00:00 2001 From: Akash Shah Date: Thu, 22 Jun 2023 22:23:04 +0530 Subject: [PATCH 22/53] Update main.yml --- ansible/roles/ml-analytics-service/defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index 95a5267c1d..a926db246d 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -123,9 +123,9 @@ ml_analytics_client_id : "{{ ml_analytics_client_id }}" ml_analytics_client_secret : "{{ ml_analytics_client_secret }}" ml_analytics_username : "{{ ml_analytics_username }}" ml_analytics_password : "{{ ml_analytics_password }}" -ml_analytics_api_base_url : "{{ ml_analytics_api_base_url }}" ml_analytics_createdBy : "{{ ml_analytics_createdBy }}" -ml_analytics_reports_container : "{{ ml_analytics_reports_container }}" -ml_analytics_reports_store : "{{ ml_analytics_reports_store }}" +ml_analytics_api_base_url : "https://{{ domain_name }}/" +ml_analytics_reports_store : "{{ cloud_service_provider }}" +ml_analytics_reports_container : "{{ cloud_storage_privatereports_bucketname }}" ml_analytics_driver_memory: "{{ ml_analytics_driver_memory | default('50g') }}" ml_analytics_executor_memory: "{{ ml_analytics_executor_memory | default('50g') }}" From 1d7ae765fffd8e786513bad3760db45082d867b2 Mon Sep 17 00:00:00 2001 From: Akash Shah Date: Mon, 26 Jun 2023 10:04:07 +0530 Subject: [PATCH 23/53] Update main.yml --- ansible/roles/ml-analytics-service/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index a926db246d..e5863e010c 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -127,5 +127,5 @@ ml_analytics_createdBy : "{{ ml_analytics_createdBy }}" ml_analytics_api_base_url : "https://{{ domain_name }}/" ml_analytics_reports_store : "{{ cloud_service_provider }}" ml_analytics_reports_container : "{{ cloud_storage_privatereports_bucketname }}" -ml_analytics_driver_memory: "{{ ml_analytics_driver_memory | default('50g') }}" -ml_analytics_executor_memory: "{{ ml_analytics_executor_memory | default('50g') }}" +ml_analytics_driver_memory: "{{ ml_analytics_driver_memory | default('5g') }}" +ml_analytics_executor_memory: "{{ ml_analytics_executor_memory | default('5g') }}" From b04cc0563b7626974fd965ab356ca44aa574368d Mon Sep 17 00:00:00 2001 From: Akash Shah Date: Mon, 26 Jun 2023 17:42:50 +0530 Subject: [PATCH 24/53] Add reference for pvt repo values --- private_repo/ansible/inventory/dev/Core/common.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/private_repo/ansible/inventory/dev/Core/common.yml b/private_repo/ansible/inventory/dev/Core/common.yml index 3af9204051..4598e3dc5d 100644 --- a/private_repo/ansible/inventory/dev/Core/common.yml +++ b/private_repo/ansible/inventory/dev/Core/common.yml @@ -475,3 +475,12 @@ sunbird_kong_device_register: 'true' # enables refersh token api call after log sunbird_kong_device_register_anonymous: 'true' # enables anonymous sessions sunbird_session_store_type: redis # uses redis for session data instead of cassandra portal_redis_connection_string: "redis://:@{{ sunbird_redis_host }}:6379/3" # Uses KP redis and DB number 3 to store session data + +#### ML Analytics Service ##### +ml_analytics_cname_url: # Reference value - https://obj.dev.sunbirded.org/samiksha/ + +### Variables added as part of 6.0.0 release - https://github.com/Sunbird-Ed/ml-analytics-service/tree/release-6.0.0/release-notes/6.0.0.md +ml_analytics_client_id : project-sunbird-dev-client # Keycloak client group +ml_analytics_username : reportAdmin1 # Username of a user which has roles of "PROGRAM_MANAGER", "PROGRAM_DESIGNER", "REPORT_ADMIN", "REPORT_VIEWER" +ml_analytics_password : Test@123 # Password of a user which has roles of "PROGRAM_MANAGER", "PROGRAM_DESIGNER", "REPORT_ADMIN", "REPORT_VIEWER" +ml_analytics_createdBy : fb85a044-d9eb-479b-a55a-faf1bfaea14d # Unique system generated user UUID which is the same user as above From 94de8aae11287a92f9aa3e8bb0dedb0a52355f36 Mon Sep 17 00:00:00 2001 From: Akash Shah Date: Mon, 26 Jun 2023 17:49:08 +0530 Subject: [PATCH 25/53] Add reference for pvt repo secret values --- private_repo/ansible/inventory/dev/Core/secrets.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/private_repo/ansible/inventory/dev/Core/secrets.yml b/private_repo/ansible/inventory/dev/Core/secrets.yml index 1107874126..68a42532b2 100644 --- a/private_repo/ansible/inventory/dev/Core/secrets.yml +++ b/private_repo/ansible/inventory/dev/Core/secrets.yml @@ -231,3 +231,9 @@ uci_odk_password: "admin" # ------------------------------------------------------------------------------------------------------------ # # Optional variables - Can be left blank if you dont plan to use the intended features lp_vault_youtube_api_key: # youtube api token if you want to upload youtube video urls on your site + +#### ML Analytics Service ##### +### Variables added as part of 6.0.0 release - https://github.com/Sunbird-Ed/ml-analytics-service/tree/release-6.0.0/release-notes/6.0.0.md +# Examples values added below +ml_analytics_authorization_access_token : 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJkOTY3NzRjYzXXXXXXXXXXXXXQ4Y2RiOWQ2Mzg0OSJ9.n4hXxKxl_698yeZPSWhXeGvMOb4esfgPadIZe8jZ0Z4' # Bearer auth token which has access to APIs as defined in the release note +ml_analytics_client_secret : fd241dce-4ZZ9-47e1-97cf-1c7de7a44216 # Keycloak client secret for creating tokens, mapped to client id supplied to variable ml_analytics_client_id From 4e59b60afb4e7ead6bc16333d0f26bc77141e083 Mon Sep 17 00:00:00 2001 From: Akash Shah Date: Mon, 26 Jun 2023 18:38:29 +0530 Subject: [PATCH 26/53] Update common.yml --- private_repo/ansible/inventory/dev/Core/common.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/private_repo/ansible/inventory/dev/Core/common.yml b/private_repo/ansible/inventory/dev/Core/common.yml index 4598e3dc5d..aef622d8cc 100644 --- a/private_repo/ansible/inventory/dev/Core/common.yml +++ b/private_repo/ansible/inventory/dev/Core/common.yml @@ -482,5 +482,4 @@ ml_analytics_cname_url: # Reference value - https://obj.dev.sunbirded.org/samiks ### Variables added as part of 6.0.0 release - https://github.com/Sunbird-Ed/ml-analytics-service/tree/release-6.0.0/release-notes/6.0.0.md ml_analytics_client_id : project-sunbird-dev-client # Keycloak client group ml_analytics_username : reportAdmin1 # Username of a user which has roles of "PROGRAM_MANAGER", "PROGRAM_DESIGNER", "REPORT_ADMIN", "REPORT_VIEWER" -ml_analytics_password : Test@123 # Password of a user which has roles of "PROGRAM_MANAGER", "PROGRAM_DESIGNER", "REPORT_ADMIN", "REPORT_VIEWER" ml_analytics_createdBy : fb85a044-d9eb-479b-a55a-faf1bfaea14d # Unique system generated user UUID which is the same user as above From 140485a08cfe94be784e2c95227050b08520d311 Mon Sep 17 00:00:00 2001 From: Akash Shah Date: Mon, 26 Jun 2023 18:39:31 +0530 Subject: [PATCH 27/53] Update secrets.yml --- private_repo/ansible/inventory/dev/Core/secrets.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/private_repo/ansible/inventory/dev/Core/secrets.yml b/private_repo/ansible/inventory/dev/Core/secrets.yml index 68a42532b2..2e16859454 100644 --- a/private_repo/ansible/inventory/dev/Core/secrets.yml +++ b/private_repo/ansible/inventory/dev/Core/secrets.yml @@ -236,4 +236,5 @@ lp_vault_youtube_api_key: # youtube api token if you want ### Variables added as part of 6.0.0 release - https://github.com/Sunbird-Ed/ml-analytics-service/tree/release-6.0.0/release-notes/6.0.0.md # Examples values added below ml_analytics_authorization_access_token : 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJkOTY3NzRjYzXXXXXXXXXXXXXQ4Y2RiOWQ2Mzg0OSJ9.n4hXxKxl_698yeZPSWhXeGvMOb4esfgPadIZe8jZ0Z4' # Bearer auth token which has access to APIs as defined in the release note +ml_analytics_password : Test@123 # Password of a user which has roles of "PROGRAM_MANAGER", "PROGRAM_DESIGNER", "REPORT_ADMIN", "REPORT_VIEWER" matching with supplied username to variable ml_analytics_username ml_analytics_client_secret : fd241dce-4ZZ9-47e1-97cf-1c7de7a44216 # Keycloak client secret for creating tokens, mapped to client id supplied to variable ml_analytics_client_id From a0a630f3561dbe0956dd4a5793f40834e8c0dc43 Mon Sep 17 00:00:00 2001 From: Akash Shah Date: Fri, 30 Jun 2023 13:04:05 +0530 Subject: [PATCH 28/53] Remove space for 6.0 added variables --- .../roles/ml-analytics-service/defaults/main.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index e5863e010c..cf896db7ae 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -119,13 +119,13 @@ ml_analytics_nvsk_imp_projects_data_blob_path: "Manage_Learn_Data/micro_improvem ml_analytics_nvsk_imp_projects_state_data_local_path: "{{ config_path }}/urgent_data_metrics/output_state/" ml_analytics_nvsk_imp_projects_state_data_blob_path: "Manage_Learn_Data/micro_improvement_state/" ml_analytics_authorization_access_token : "{{ ml_analytics_authorization_access_token }}" -ml_analytics_client_id : "{{ ml_analytics_client_id }}" -ml_analytics_client_secret : "{{ ml_analytics_client_secret }}" -ml_analytics_username : "{{ ml_analytics_username }}" -ml_analytics_password : "{{ ml_analytics_password }}" -ml_analytics_createdBy : "{{ ml_analytics_createdBy }}" -ml_analytics_api_base_url : "https://{{ domain_name }}/" -ml_analytics_reports_store : "{{ cloud_service_provider }}" -ml_analytics_reports_container : "{{ cloud_storage_privatereports_bucketname }}" +ml_analytics_client_id: "{{ ml_analytics_client_id }}" +ml_analytics_client_secret: "{{ ml_analytics_client_secret }}" +ml_analytics_username: "{{ ml_analytics_username }}" +ml_analytics_password: "{{ ml_analytics_password }}" +ml_analytics_createdBy: "{{ ml_analytics_createdBy }}" +ml_analytics_api_base_url: "https://{{ domain_name }}/" +ml_analytics_reports_store: "{{ cloud_service_provider }}" +ml_analytics_reports_container: "{{ cloud_storage_privatereports_bucketname }}" ml_analytics_driver_memory: "{{ ml_analytics_driver_memory | default('5g') }}" ml_analytics_executor_memory: "{{ ml_analytics_executor_memory | default('5g') }}" From c05302f03f43e49082355a599c0d534105452299 Mon Sep 17 00:00:00 2001 From: Akash Shah Date: Fri, 30 Jun 2023 14:11:43 +0530 Subject: [PATCH 29/53] Remove extra spaces --- ansible/roles/ml-analytics-service/defaults/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index cf896db7ae..c979561d7b 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -53,8 +53,8 @@ ml_analytics_project_output_dir: "{{ WORKDIR }}/source/projects/output" ml_analytics_observation_status_output_dir: "{{ WORKDIR }}/source/observations/status/output" ml_analytics_api_authorization_key: "{{ml_api_auth_token | default('sunbird_api_auth_token')}}" ml_analytics_api_access_token: "{{ml_api_access_token | default('ml_core_internal_access_token')}}" -ml_analytics_druid_observation_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/observation/status/sl_observation_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_code","school_externalId","state_name","state_code","state_externalId","district_name","district_code","district_externalId","block_name","block_code","block_externalId","cluster_name","cluster_code","cluster_externalId","completedDate","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name","ecm_marked_na","board_name","updatedAt","organisation_id","user_type","observed_school_name","observed_school_id","observed_school_code","observed_state_name","observed_state_id","observed_state_code","observed_district_name","observed_district_id","observed_district_code","observed_block_name","observed_block_id","observed_block_code","observed_cluster_name","observed_cluster_id","observed_cluster_code","isRubricDriven","criteriaLevelReport"]},"metricsSpec":[]}}}' -ml_analytics_druid_project_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/projects/sl_projects.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-project","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"auto"},"dimensionsSpec":{"dimensions":[]},"metricsSpec":[]}}}' +ml_analytics_druid_observation_status_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/observation/status/sl_observation_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_code","school_externalId","state_name","state_code","state_externalId","district_name","district_code","district_externalId","block_name","block_code","block_externalId","cluster_name","cluster_code","cluster_externalId","completedDate","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name","ecm_marked_na","board_name","updatedAt","organisation_id","user_type","observed_school_name","observed_school_id","observed_school_code","observed_state_name","observed_state_id","observed_state_code","observed_district_name","observed_district_id","observed_district_code","observed_block_name","observed_block_id","observed_block_code","observed_cluster_name","observed_cluster_id","observed_cluster_code","isRubricDriven","criteriaLevelReport"]},"metricsSpec":[]}}}' +ml_analytics_druid_project_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/projects/sl_projects.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-project","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"auto"},"dimensionsSpec":{"dimensions":[]},"metricsSpec":[]}}}' ml_analytics_druid_distinctCnt_obs_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount/ml_observation_distinctCount_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' ml_analytics_druid_distinctCnt_obs_domain_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain/ml_observation_distinctCount_domain.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain_criteria/ml_observation_distinctCount_domain_criteria.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain-criteria","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"string","name":"criteria_name"},{"type":"string","name":"criteria_score"},{"type":"string","name":"criteria_id"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' @@ -81,7 +81,7 @@ ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec : '{"type ml_analytics_projects_distinctCnt_prglevel_output_dir: "{{ WORKDIR }}/source/projects/distinctCountPrglevel/output" ml_analytics_projects_distinctCnt_prglevel_cloud_blob_path: "projects/distinctCountPrglevel/" ml_analytics_survey_status_output_dir : "{{ WORKDIR }}/source/survey/status/output" -ml_analytics_survey_cloud_blob_path : "survey/status/" +ml_analytics_survey_cloud_blob_path: "survey/status/" ml_analytics_druid_survey_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/survey/status/sl_survey_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel", "solution_name", "solution_id","private_program","state_code","school_code","district_code","block_code","cluster_code"]},"metricsSpec":[]}}}' ml_slack_token: "{{ ml_analytics_slack_token | default('') }}" ml_slack_channel: "{{ ml_analytics_slack_channel | default('') }}" @@ -96,7 +96,7 @@ ml_analytics_observation_submission_id_filepath: "{{ WORKDIR }}/ml-analytics-ser ml_analytics_observation_batchupdate_output_dir: "{{ WORKDIR }}/source/observations/" ml_analytics_druid_survey_query_spec : '{"queryType":"scan","dataSource":"sl-survey","resultFormat":"list","columns":["completedDate","createdAt","createdBy","criteriaExternalId","criteriaId","criteriaName","surveyId","surveyName","surveySubmissionId","questionAnswer","questionECM","questionExternalId","questionId","questionName","questionResponseLabel","questionResponseType","solutionExternalId","solutionId","solutionName","updatedAt","instanceParentId","instanceId","instanceParentResponsetype","instanceParentQuestion","questionSequenceByEcm","maxScore","minScore","percentageScore","pointsBasedScoreInParent","totalScore","scoreAchieved","totalpercentage","instanceParentExternalId","instanceParentEcmSequence","remarks","total_evidences","evidence_count","instanceParentCriteriaId","instanceParentCriteriaExternalId","instanceParentCriteriaName","isAPrivateProgram","programId","programName","programExternalId","questionResponseLabel_number","channel","parent_channel","appName","organisation_name","user_subtype","user_type","board_name","district_code","district_name","district_externalId","block_code","block_name","block_externalId","school_code","school_name","school_externalId","cluster_code","cluster_name","cluster_externalId","state_code","state_name","state_externalId","organisation_id","evidences"],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]}' ml_analytics_druid_survey_batch_ingestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris": ["azure://telemetry-data-store/survey/batchDeletion/druidData.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-survey","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"surveyId"},{"type":"string","name":"surveyName"},{"type":"string","name":"surveySubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"evidences"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"appName"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_subtype"},{"type":"string","name":"user_type"},{"type":"string","name":"board_name"},{"type":"string","name":"district_code"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_code"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"school_code"},{"type":"string","name":"school_name"},{"type":"string","name":"school_externalId"},{"type":"string","name":"cluster_code"},{"type":"string","name":"cluster_name"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"state_code"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}' -ml_analytics_survey_batchupdate_cloud_blob_path : "survey/batchDeletion" +ml_analytics_survey_batchupdate_cloud_blob_path: "survey/batchDeletion" ml_analytics_survey_submission_id_filepath : "{{ WORKDIR }}/ml-analytics-service/survey/submissions.csv" ml_analytics_survey_batchupdate_output_dir : "{{ WORKDIR }}/source/survey/" ml_analytics_druid_interval_list: '["1901-01-01T00:00:00+00:00/2020-01-01T00:00:00+00:00","2020-01-01T00:00:00+00:00/2020-06-01T00:00:00+00:00","2020-06-01T00:00:00+00:00/2021-01-01T00:00:00+00:00","2021-01-01T00:00:00+00:00/2021-06-01T00:00:00+00:00","2021-06-01T00:00:00+00:00/2022-01-01T00:00:00+00:00","2022-01-01T00:00:00+00:00/2022-03-01T00:00:00+00:00","2022-03-01T00:00:00+00:00/2022-06-01T00:00:00+00:00","2022-06-01T00:00:00+00:00/2022-09-01T00:00:00+00:00","2022-09-01T00:00:00+00:00/2023-01-01T00:00:00+00:00"]' @@ -118,7 +118,7 @@ ml_analytics_nvsk_imp_projects_data_local_path: "{{ config_path }}/urgent_data_m ml_analytics_nvsk_imp_projects_data_blob_path: "Manage_Learn_Data/micro_improvement/" ml_analytics_nvsk_imp_projects_state_data_local_path: "{{ config_path }}/urgent_data_metrics/output_state/" ml_analytics_nvsk_imp_projects_state_data_blob_path: "Manage_Learn_Data/micro_improvement_state/" -ml_analytics_authorization_access_token : "{{ ml_analytics_authorization_access_token }}" +ml_analytics_authorization_access_token: "{{ ml_analytics_authorization_access_token }}" ml_analytics_client_id: "{{ ml_analytics_client_id }}" ml_analytics_client_secret: "{{ ml_analytics_client_secret }}" ml_analytics_username: "{{ ml_analytics_username }}" From 6dfd079db31c674968f649cc1a5126b9681f6310 Mon Sep 17 00:00:00 2001 From: Akash Shah Date: Fri, 30 Jun 2023 14:14:29 +0530 Subject: [PATCH 30/53] Update main.yml --- ansible/roles/ml-analytics-service/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index c979561d7b..881afcbd41 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -97,8 +97,8 @@ ml_analytics_observation_batchupdate_output_dir: "{{ WORKDIR }}/source/observati ml_analytics_druid_survey_query_spec : '{"queryType":"scan","dataSource":"sl-survey","resultFormat":"list","columns":["completedDate","createdAt","createdBy","criteriaExternalId","criteriaId","criteriaName","surveyId","surveyName","surveySubmissionId","questionAnswer","questionECM","questionExternalId","questionId","questionName","questionResponseLabel","questionResponseType","solutionExternalId","solutionId","solutionName","updatedAt","instanceParentId","instanceId","instanceParentResponsetype","instanceParentQuestion","questionSequenceByEcm","maxScore","minScore","percentageScore","pointsBasedScoreInParent","totalScore","scoreAchieved","totalpercentage","instanceParentExternalId","instanceParentEcmSequence","remarks","total_evidences","evidence_count","instanceParentCriteriaId","instanceParentCriteriaExternalId","instanceParentCriteriaName","isAPrivateProgram","programId","programName","programExternalId","questionResponseLabel_number","channel","parent_channel","appName","organisation_name","user_subtype","user_type","board_name","district_code","district_name","district_externalId","block_code","block_name","block_externalId","school_code","school_name","school_externalId","cluster_code","cluster_name","cluster_externalId","state_code","state_name","state_externalId","organisation_id","evidences"],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]}' ml_analytics_druid_survey_batch_ingestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris": ["azure://telemetry-data-store/survey/batchDeletion/druidData.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-survey","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"surveyId"},{"type":"string","name":"surveyName"},{"type":"string","name":"surveySubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"evidences"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"appName"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_subtype"},{"type":"string","name":"user_type"},{"type":"string","name":"board_name"},{"type":"string","name":"district_code"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_code"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"school_code"},{"type":"string","name":"school_name"},{"type":"string","name":"school_externalId"},{"type":"string","name":"cluster_code"},{"type":"string","name":"cluster_name"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"state_code"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}' ml_analytics_survey_batchupdate_cloud_blob_path: "survey/batchDeletion" -ml_analytics_survey_submission_id_filepath : "{{ WORKDIR }}/ml-analytics-service/survey/submissions.csv" -ml_analytics_survey_batchupdate_output_dir : "{{ WORKDIR }}/source/survey/" +ml_analytics_survey_submission_id_filepath: "{{ WORKDIR }}/ml-analytics-service/survey/submissions.csv" +ml_analytics_survey_batchupdate_output_dir: "{{ WORKDIR }}/source/survey/" ml_analytics_druid_interval_list: '["1901-01-01T00:00:00+00:00/2020-01-01T00:00:00+00:00","2020-01-01T00:00:00+00:00/2020-06-01T00:00:00+00:00","2020-06-01T00:00:00+00:00/2021-01-01T00:00:00+00:00","2021-01-01T00:00:00+00:00/2021-06-01T00:00:00+00:00","2021-06-01T00:00:00+00:00/2022-01-01T00:00:00+00:00","2022-01-01T00:00:00+00:00/2022-03-01T00:00:00+00:00","2022-03-01T00:00:00+00:00/2022-06-01T00:00:00+00:00","2022-06-01T00:00:00+00:00/2022-09-01T00:00:00+00:00","2022-09-01T00:00:00+00:00/2023-01-01T00:00:00+00:00"]' ML_Cloud_Service_Provider: "{{ cloud_service_provider | default('azure') }}" ## Valid options - ORACLE, gcloud, aws & azure ml_analytics_cloud_package_path: "{{ config_path }}/cloud_storage" From f2f6a80fef4ff7e33ed1577d849626076416f27c Mon Sep 17 00:00:00 2001 From: Akash Shah Date: Fri, 30 Jun 2023 18:09:56 +0530 Subject: [PATCH 31/53] Update main.yml --- ansible/roles/ml-analytics-service/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index 881afcbd41..d62a4cfd31 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -127,5 +127,5 @@ ml_analytics_createdBy: "{{ ml_analytics_createdBy }}" ml_analytics_api_base_url: "https://{{ domain_name }}/" ml_analytics_reports_store: "{{ cloud_service_provider }}" ml_analytics_reports_container: "{{ cloud_storage_privatereports_bucketname }}" -ml_analytics_driver_memory: "{{ ml_analytics_driver_memory | default('5g') }}" -ml_analytics_executor_memory: "{{ ml_analytics_executor_memory | default('5g') }}" +ml_analytics_driver_memory: "{{ ml_analytics_default_driver_memory | default('5g') }}" +ml_analytics_executor_memory: "{{ ml_analytics_default_executor_memory | default('5g') }}" From ac256eabcc445480eb0b55a42c24d88a72629225 Mon Sep 17 00:00:00 2001 From: sachin patil <91308046+spsac08@users.noreply.github.com> Date: Mon, 3 Jul 2023 17:13:15 +0530 Subject: [PATCH 32/53] Update config.j2 Updated the Loggers path in config.j2 --- .../ml-analytics-service/templates/config.j2 | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/ansible/roles/ml-analytics-service/templates/config.j2 b/ansible/roles/ml-analytics-service/templates/config.j2 index ad762488c6..e15c7eb70b 100644 --- a/ansible/roles/ml-analytics-service/templates/config.j2 +++ b/ansible/roles/ml-analytics-service/templates/config.j2 @@ -131,29 +131,19 @@ survey_druid_topic = {{ ml_analytics_kafka_survey_druid_topic_name }} [LOGS] -observation_streaming_success = {{ ml_analytics_observation_log_folder_path }}/success.log +observation_streaming_success_error = {{ ml_analytics_observation_log_folder_path }}/ -observation_streaming_error = {{ ml_analytics_observation_log_folder_path }}/error.log +observation_streaming_evidence_success_error = {{ ml_analytics_observation_log_folder_path }}/evidence/ -observation_streaming_evidence_success = {{ ml_analytics_observation_log_folder_path }}/evidence/success.log +observation_status_success_error = {{ ml_analytics_observation_log_folder_path }}/status/ -observation_streaming_evidence_error = {{ ml_analytics_observation_log_folder_path }}/evidence/error.log +project_success_error = {{ ml_analytics_project_log_folder_path }}/ -observation_status_success = {{ ml_analytics_observation_log_folder_path }}/status/success.log +survey_evidence_streaming_success_error = {{ ml_analytics_survey_log_folder_path }}/evidence/ -observation_status_error = {{ ml_analytics_observation_log_folder_path }}/status/error.log +survey_streaming_success_error = {{ ml_analytics_survey_log_folder_path }}/ -project_success = {{ ml_analytics_project_log_folder_path }}/success.log - -project_error = {{ ml_analytics_project_log_folder_path }}/error.log - -survey_evidence_streaming_success = {{ ml_analytics_survey_log_folder_path }}/evidence/success.log - -survey_evidence_streaming_error = {{ ml_analytics_survey_log_folder_path }}/evidence/error.log - -survey_streaming_success = {{ ml_analytics_survey_log_folder_path }}/success.log - -survey_streaming_error = {{ ml_analytics_survey_log_folder_path }}/error.log +survey_status_success_error = {{ ml_analytics_survey_log_folder_path }}/status/ {% if ML_Cloud_Service_Provider is equalto 'ORACLE' %} From d116ed41bf8db0f5cad3201465e169bb313803f0 Mon Sep 17 00:00:00 2001 From: Rajesh Kumaravel Date: Mon, 10 Jul 2023 11:29:54 +0530 Subject: [PATCH 33/53] Issue #ED-2580 fix: Update default tenant --- .../sunbird/login/resources/js/telemetry_service.js | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js b/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js index e858bfb817..6d2e91f79b 100644 --- a/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js +++ b/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js @@ -2477,19 +2477,14 @@ if (typeof module != 'undefined') { let pdataId = ""; let tenantSlug; -if (window.location.origin.indexOf("diksha.gov.in") >= 0) { - pdataId = "prod.diksha.portal"; -} else if (window.location.origin.indexOf("staging.ntp.net.in") >= 0) { - pdataId = "staging.diksha.portal"; -} else if (window.location.origin.indexOf("staging.sunbirded.org") >= 0) { +if (window.location.origin.indexOf("staging.sunbirded.org") >= 0) { pdataId = "staging.diksha.portal"; } else if (window.location.origin.indexOf("dev.sunbirded.org") >= 0) { pdataId = "dev.sunbird.portal"; tenantSlug = "sunbird"; } else { - pdataId = "preprod.diksha.portal"; tenantSlug = "sunbird"; - + pdataId = "dev.sunbird.portal"; } let curUrlObj = window.location; From d83a3484624cb32d32861b45852bac58074b6073 Mon Sep 17 00:00:00 2001 From: Rajesh Kumaravel Date: Mon, 10 Jul 2023 12:02:24 +0530 Subject: [PATCH 34/53] Issue #ED-2580 fix: Update default tenant --- .../artifacts/sunbird/login/resources/js/telemetry_service.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js b/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js index 6d2e91f79b..5b4b6c6125 100644 --- a/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js +++ b/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js @@ -2530,7 +2530,7 @@ if(client_id.toLowerCase() === 'android'){ request: { filters: { isTenant: true, - slug: id || 'ntp' + slug: id || 'sunbird' } } }), From 8494c44bceff86514a2ae4846d3b14f1c7d0e15a Mon Sep 17 00:00:00 2001 From: Rajeev Satish Date: Wed, 19 Jul 2023 11:28:26 +0530 Subject: [PATCH 35/53] Issue fix ED-2586 'Diksha' text is appearing in help page and in SSO login --- ansible/artifacts/sunbird/login/messages/messages_en.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/artifacts/sunbird/login/messages/messages_en.properties b/ansible/artifacts/sunbird/login/messages/messages_en.properties index 311b7512a0..9e66019f5b 100644 --- a/ansible/artifacts/sunbird/login/messages/messages_en.properties +++ b/ansible/artifacts/sunbird/login/messages/messages_en.properties @@ -56,7 +56,7 @@ noAccount=Don''t have an account? username=Username or Mobile number goBack=< Date: Wed, 26 Jul 2023 10:12:46 +0530 Subject: [PATCH 36/53] [ED-2164] Release 6.0.0 changes (#3840) * Add missing var * ED-2164: Add new var assessment service - #IQ-329 * Update dial service search var * Fix sytax issue * Onboard new apis lern-release-5.3.0 --- ansible/roles/kong-api/defaults/main.yml | 66 +++++++++++++++++++ .../assessment-service_application.conf | 5 +- .../templates/sunbird_lms-service.env | 4 +- .../inventory/dev/DataPipeline/common.yml | 5 +- 4 files changed, 75 insertions(+), 5 deletions(-) diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index c59e52bf17..6090bfff0f 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -10066,3 +10066,69 @@ kong_apis: - name: opa-checks config.required: false config.enabled: false + +#Lern release-5.3.0 + +- name: exhaustSubmitProxyAPI + uris: "{{ course_service_prefix }}/v1/jobrequest/submit" + upstream_url: "{{ lms_service_url }}/v1/jobrequest/submit" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - courseAccess + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: exhaustListProxyAPI + uris: "{{ course_service_prefix }}/v1/jobrequest/list" + upstream_url: "{{ lms_service_url }}/v1/jobrequest/list" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - courseAccess + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: orgAddEncryptionKey + uris: "{{ org_service_prefix }}/v1/update/encryptionkey" + upstream_url: "{{ learning_service_url }}/v1/org/update/encryptionkey" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - orgSuperAdmin + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false +#Lern release-5.3.0 diff --git a/ansible/roles/stack-sunbird/templates/assessment-service_application.conf b/ansible/roles/stack-sunbird/templates/assessment-service_application.conf index c06a44f2d8..d5e7723a19 100644 --- a/ansible/roles/stack-sunbird/templates/assessment-service_application.conf +++ b/ansible/roles/stack-sunbird/templates/assessment-service_application.conf @@ -429,4 +429,7 @@ cloudstorage { relative_path_prefix="{{ cloudstorage_relative_path_prefix | default('CLOUD_STORAGE_BASE_PATH') }}" read_base_path="{{ cloudstorage_base_path }}" write_base_path={{ valid_cloudstorage_base_urls }} -} \ No newline at end of file +} + +#Inquiry-release-5.6.0 +question.list.limit={{ question_list_api_request_limit | default('20') }} diff --git a/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env b/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env index a1682a6971..0318a4ed7e 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env @@ -154,5 +154,5 @@ cloud_store_base_path_placeholder={{ cloud_store_base_path_placeholder | default # Release-5.2.0 LR-325 -sunbird_dial_service_base_url = {{ dial_service_url | default('http://dial-service:9000') }} -sunbird_dial_service_search_url = "/v1/dialcode/search" \ No newline at end of file +sunbird_dial_service_base_url={{ dial_service_url | default('http://dial-service:9000') }} +sunbird_dial_service_search_url=/dialcode/v3/search diff --git a/private_repo/ansible/inventory/dev/DataPipeline/common.yml b/private_repo/ansible/inventory/dev/DataPipeline/common.yml index 09431526ea..971c04935c 100644 --- a/private_repo/ansible/inventory/dev/DataPipeline/common.yml +++ b/private_repo/ansible/inventory/dev/DataPipeline/common.yml @@ -3,8 +3,9 @@ cloud_service_provider: "" # Your cloud service provider name. Supported values are aws, azure, gcloud domain_name: "" # your domain name like example.com # docker hub details -dockerhub: "change.docker.url" # docker hub username or url incase of private registry -private_ingressgateway_ip: "" # your private kubernetes load balancer ip +dockerhub: "change.docker.url" # docker hub username or url incase of private registry +private_ingressgateway_ip: "" # your private kubernetes load balancer ip +search_lb_ip: # search service Load balancer IP # Cloud Service Provider Variables # If cloud_service_provider is AWS then update with access key as value From 4df16ffa522f80096c0ae141490bfc00c74ec305 Mon Sep 17 00:00:00 2001 From: Abhishek P N <116337484+abhishekpnt@users.noreply.github.com> Date: Wed, 9 Aug 2023 17:04:03 +0530 Subject: [PATCH 37/53] Issue #ED-2632 fix: removed unused env variables and added OCI endpoint variable (#3876) * Issue #ED-2632 fix: removed unused env variables and added OCI endpoint variable * Issue #ED-2632 cleanup: Removed Azure related variables * Issue #ED-2632 cleanup: changed added storage account url --- .../templates/sunbird_player.env | 27 +++---------------- 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/ansible/roles/stack-sunbird/templates/sunbird_player.env b/ansible/roles/stack-sunbird/templates/sunbird_player.env index 6c2c86f19f..6486898185 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_player.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_player.env @@ -15,8 +15,6 @@ sunbird_api_auth_token={{sunbird_api_auth_token}} sunbird_enable_permission_check=1 sunbird_cassandra_urls={{sunbird_cassandra_urls}} sunbird_session_store_type={{sunbird_session_store_type}} -AZURE_STORAGE_ACCOUNT={{sunbird_public_storage_account_name}} -AZURE_STORAGE_KEY={{sunbird_public_storage_account_key}} sunbird_dataservice_url={{sunbird_dataservice_url}} sunbird_telemetry_packet_size={{sunbird_telemetry_packet_size}} sunbird_environment={{sunbird_environment}} @@ -42,9 +40,6 @@ sunbird_google_captcha_site_key={{sunbird_google_captcha_site_key}} sunbird_google_keycloak_client_id={{sunbird_google_keycloak_client_id}} sunbird_google_keycloak_secret={{sunbird_google_keycloak_secret}} sunbird_help_link_visibility={{sunbird_help_link_visibility|default('FALSE')}} -sunbird_azure_report_container_name={{sunbird_azure_report_container_name}} -sunbird_azure_account_name={{sunbird_private_storage_account_name}} -sunbird_azure_account_key={{sunbird_private_storage_account_key}} sunbird_response_cache_ttl={{sunbird_response_cache_ttl}} sunbird_portal_preview_cdn_url={{sunbird_portal_preview_cdn_url}} @@ -162,7 +157,7 @@ sunbird_base_proto={{sunbird_base_proto | default(proto)}} ml_survey_url={{ml_survey_url|default(proto+'://' +domain_name) }} #Release-4.1.0 -sunbird_azure_storage_account_name={{ cloud_storage_url }}/ +sunbird_public_storage_account_name={{ cloud_storage_url }}/ #Release-4.1.0 sunbird_google_oauth_ios_clientId={{sunbird_google_oauth_ios_clientId | default("")}} @@ -205,27 +200,10 @@ sunbird_questionset_children_limit={{ sunbird_questionset_children_limit | defau #release-5.0.0 -#AWS config for storage access -#To be reverted post testing sunbird_cloud_storage_provider={{cloud_service_provider}} -sunbird_aws_bucket_name={{aws_private_bucket_name | default("")}} -sunbird_aws_reports={{aws_reports_folder_name | default("reports")}} -sunbird_aws_labels={{aws_labels_folder_name | default("labels")}} -sunbird_aws_access_key={{aws_private_bucket_access_key | default("")}} -sunbird_aws_secret_key={{aws_private_bucket_secret_key | default("")}} -sunbird_aws_region={{aws_private_bucket_region | default("ap-south-1")}} - -#GCLOUD config for storage access -sunbird_gcloud_bucket_name={{gcloud_private_bucket_name | default("")}} -sunbird_gcloud_reports={{gcloud_reports_folder_name | default("reports")}} -sunbird_gcloud_labels={{gcloud_labels_folder_name | default("labels")}} -sunbird_gcloud_client_email={{gcloud_private_bucket_client_email | default("")}} -sunbird_gcloud_private_key={{gcloud_private_bucket_private_key | default("")}} -sunbird_gcloud_projectId={{gcloud_private_bucket_projectId | default("")}} #release-5.1.0 #CSP configuration variables changes -cloud_service_provider={{cloud_service_provider}} cloud_private_storage_accountname={{cloud_private_storage_accountname | default("")}} cloud_private_storage_secret={{cloud_private_storage_secret | default("")}} cloud_private_storage_region={{cloud_private_storage_region | default("ap-south-1")}} @@ -235,4 +213,5 @@ cloud_storage_resourceBundle_bucketname={{cloud_storage_label_bucketname | defau cloud_storage_desktopCrash_bucketname={{cloud_storage_desktopcrash_bucketname | default("desktopappcrashlogs")}} #release-6.0.0 -sunbird_default_board={{sunbird_default_board | default("")}} \ No newline at end of file +sunbird_default_board={{sunbird_default_board | default("")}} +cloud_private_storage_endpoint={{cloud_private_storage_endpoint | default("")}} \ No newline at end of file From da25f005379da758d44d1a3822bddb230dbce01b Mon Sep 17 00:00:00 2001 From: VISHNUDAS <95604247+VISHNUDAS-tunerlabs@users.noreply.github.com> Date: Mon, 14 Aug 2023 15:54:12 +0530 Subject: [PATCH 38/53] ENV changes related to CSP (#3878) * envChange CSP(Azure) * CSP env changes * removed csp references * env changes for core service * default added * added all variables * env variable name change * ENV changes * Changes to env variables --- .../templates/ml-core-service.env | 45 ++++--------------- 1 file changed, 9 insertions(+), 36 deletions(-) diff --git a/ansible/roles/stack-sunbird/templates/ml-core-service.env b/ansible/roles/stack-sunbird/templates/ml-core-service.env index 0209e84ae6..515495006a 100755 --- a/ansible/roles/stack-sunbird/templates/ml-core-service.env +++ b/ansible/roles/stack-sunbird/templates/ml-core-service.env @@ -19,42 +19,15 @@ INTERNAL_ACCESS_TOKEN={{ml_core_internal_access_token}} #Gotenberg configuration GOTENBERG_SERVICE_URL={{ml_gotenberg_url | default("http://gotenberg-service:3000")}} -## Cloud Environment platform -# which will be templatized from the above variable (default) - -# Cloud configuration for assesment service -# Options available -# -## Azure Cloud Configuration -# -# CLOUD_STORAGE=AZURE -# AZURE_ACCOUNT_NAME= -# AZURE_ACCOUNT_KEY= -# AZURE_STORAGE_CONTAINER= -# -## Google Cloud Configuration -# -# CLOUD_STORAGE=GCP -# GCP_PATH= -# GCP_BUCKET_NAME= -# -## AWS Cloud Configuration -# -# CLOUD_STORAGE=AWS -# AWS_ACCESS_KEY_ID= -# AWS_SECRET_ACCESS_KEY= -# AWS_BUCKET_NAME= -# AWS_BUCKET_ENDPOINT= -# AWS_BUCKET_REGION= -# To change the cloud config for azure, override below variable in common.yaml -# For Example, -# ml_survey_cloud_config: | -# CLOUD_STORAGE=AZURE -# AZURE_ACCOUNT_NAME=mystorageaccount -# AZURE_ACCOUNT_KEY=xx123supersecretpassword -# AZURE_STORAGE_CONTAINER=slassessmentscontainer - -{{ml_cloud_config}} + +# CLOUD STORAGE +CLOUD_STORAGE_PROVIDER={{sunbird_cloud_storage_provider}} +CLOUD_STORAGE_ACCOUNTNAME={{cloud_private_storage_accountname | default("")}} +CLOUD_STORAGE_SECRET={{cloud_private_storage_secret | default("")}} +CLOUD_STORAGE_BUCKETNAME={{ml_container | default("")}} +CLOUD_STORAGE_REGION={{cloud_private_storage_region | default("")}} +CLOUD_STORAGE_PROJECT={{cloud_private_storage_project | default("")}} +CLOUD_ENDPOINT={{cloud_private_storage_endpoint | default("")}} # OFFLINE TOKEN VALIDATION From 100ef92cd237d67203c6b1ee974b2a0634f43b49 Mon Sep 17 00:00:00 2001 From: VISHNUDAS <95604247+VISHNUDAS-tunerlabs@users.noreply.github.com> Date: Wed, 16 Aug 2023 12:56:24 +0530 Subject: [PATCH 39/53] provider variable name change (#3880) --- ansible/roles/stack-sunbird/templates/ml-core-service.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/stack-sunbird/templates/ml-core-service.env b/ansible/roles/stack-sunbird/templates/ml-core-service.env index 515495006a..30018c5cd8 100755 --- a/ansible/roles/stack-sunbird/templates/ml-core-service.env +++ b/ansible/roles/stack-sunbird/templates/ml-core-service.env @@ -21,7 +21,7 @@ GOTENBERG_SERVICE_URL={{ml_gotenberg_url | default("http://gotenberg-service:300 # CLOUD STORAGE -CLOUD_STORAGE_PROVIDER={{sunbird_cloud_storage_provider}} +CLOUD_STORAGE_PROVIDER={{cloud_service_provider}} CLOUD_STORAGE_ACCOUNTNAME={{cloud_private_storage_accountname | default("")}} CLOUD_STORAGE_SECRET={{cloud_private_storage_secret | default("")}} CLOUD_STORAGE_BUCKETNAME={{ml_container | default("")}} From f484132a869fa4f97d6d8f3c455282727f18cee3 Mon Sep 17 00:00:00 2001 From: sachin patil <91308046+spsac08@users.noreply.github.com> Date: Wed, 16 Aug 2023 14:28:41 +0530 Subject: [PATCH 40/53] #ED-1383 Release 6.0.0 (#3875) * Update config.j2 Adding new variables for Survey data sources * Update main.yml Adding Variables * Druid injestion spec update --- .../ml-analytics-service/defaults/main.yml | 29 ++++++++++--------- .../ml-analytics-service/templates/config.j2 | 6 ++++ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index d62a4cfd31..98be525421 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -53,12 +53,13 @@ ml_analytics_project_output_dir: "{{ WORKDIR }}/source/projects/output" ml_analytics_observation_status_output_dir: "{{ WORKDIR }}/source/observations/status/output" ml_analytics_api_authorization_key: "{{ml_api_auth_token | default('sunbird_api_auth_token')}}" ml_analytics_api_access_token: "{{ml_api_access_token | default('ml_core_internal_access_token')}}" -ml_analytics_druid_observation_status_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/observation/status/sl_observation_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_code","school_externalId","state_name","state_code","state_externalId","district_name","district_code","district_externalId","block_name","block_code","block_externalId","cluster_name","cluster_code","cluster_externalId","completedDate","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name","ecm_marked_na","board_name","updatedAt","organisation_id","user_type","observed_school_name","observed_school_id","observed_school_code","observed_state_name","observed_state_id","observed_state_code","observed_district_name","observed_district_id","observed_district_code","observed_block_name","observed_block_id","observed_block_code","observed_cluster_name","observed_cluster_id","observed_cluster_code","isRubricDriven","criteriaLevelReport"]},"metricsSpec":[]}}}' -ml_analytics_druid_project_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/projects/sl_projects.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-project","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"auto"},"dimensionsSpec":{"dimensions":[]},"metricsSpec":[]}}}' -ml_analytics_druid_distinctCnt_obs_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount/ml_observation_distinctCount_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' -ml_analytics_druid_distinctCnt_obs_domain_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain/ml_observation_distinctCount_domain.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' -ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain_criteria/ml_observation_distinctCount_domain_criteria.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain-criteria","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"string","name":"criteria_name"},{"type":"string","name":"criteria_score"},{"type":"string","name":"criteria_id"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' -ml_analytics_druid_distinctCnt_projects_status_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/distinctCount/ml_projects_distinctCount.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"project_title"},{"type":"string","name":"solution_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"long","name":"no_of_imp_with_evidence_inprogress"},{"type":"string","name":"time_stamp"},{"type":"long","name":"no_of_certificate_issued"}]},"metricsSpec":[]}}}' +ml_analytics_druid_observation_status_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_code","school_externalId","state_name","state_code","state_externalId","district_name","district_code","district_externalId","block_name","block_code","block_externalId","cluster_name","cluster_code","cluster_externalId","completedDate","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name","ecm_marked_na","board_name","updatedAt","organisation_id","user_type","observed_school_name","observed_school_id","observed_school_code","observed_state_name","observed_state_id","observed_state_code","observed_district_name","observed_district_id","observed_district_code","observed_block_name","observed_block_id","observed_block_code","observed_cluster_name","observed_cluster_id","observed_cluster_code","isRubricDriven","criteriaLevelReport"]},"metricsSpec":[]}}}' +ml_analytics_druid_project_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-project","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"auto"},"dimensionsSpec":{"dimensions":[]},"metricsSpec":[]}}}' +ml_analytics_druid_distinctCnt_obs_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' +ml_analytics_druid_distinctCnt_obs_domain_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' +ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain-criteria","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"string","name":"criteria_name"},{"type":"string","name":"criteria_score"},{"type":"string","name":"criteria_id"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' +ml_analytics_druid_distinctCnt_projects_status_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"project_title"},{"type":"string","name":"solution_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"long","name":"no_of_imp_with_evidence_inprogress"},{"type":"string","name":"time_stamp"},{"type":"long","name":"no_of_certificate_issued"}]},"metricsSpec":[]}}}' +ml_analytics_druid_distinctCnt_survey_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-surveydistinctCount-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"survey_name"},{"type":"string","name":"survey_id"},{"type":"string","name":"submission_status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_submissions"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' ml_analytics_obs_distinctCnt_cloud_blob_path: "observation/distinctCount/" ml_analytics_obs_distinctCnt_domain_cloud_blob_path: "observation/distinctCount_domain/" ml_analytics_obs_distinctCnt_domain_criteria_cloud_blob_path: "observation/distinctCount_domain_criteria/" @@ -67,22 +68,24 @@ ml_analytics_obs_distinctCnt_output_dir: "{{ WORKDIR }}/source/observations/dist ml_analytics_obs_distinctCnt_domain_output_dir: "{{ WORKDIR }}/source/observations/distinctCount_domain/output" ml_analytics_obs_distinctCnt_domain_criteria_output_dir: "{{ WORKDIR }}/source/observations/distinctCount_domain_criteria/output" ml_analytics_projects_distinctCnt_output_dir: "{{ WORKDIR }}/source/projects/distinctCount/output" +ml_analytics_survey_distinctCnt_output_dir: "{{ WORKDIR }}/source/survey/distinctCount/output" ml_analytics_survey_rollup_cloud_blob_path: "survey/rollup/" -ml_analytics_druid_survey_rollup_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/survey/rollup/sl_survey_rollup.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel",{"type":"long","name":"status_code"}, "solution_name", "solution_id"]},"metricsSpec":[{"name":"count","type":"count"},{"name":"sum___v","type":"longSum","fieldName":"__v"},{"name":"sum_status_code","type":"longMax","fieldName":"status_code"},{"type":"HLLSketchBuild","name":"count_of_user_id","fieldName":"user_id"},{"type":"HLLSketchBuild","name":"count_of_survey_submission_id","fieldName":"survey_submission_id"},{"type":"HLLSketchBuild","name":"count_of_solution_id","fieldName":"solution_id"},{"type":"HLLSketchBuild","name":"count_of_program_id","fieldName":"program_id"}]}}}' +ml_analytics_druid_survey_rollup_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel",{"type":"long","name":"status_code"}, "solution_name", "solution_id"]},"metricsSpec":[{"name":"count","type":"count"},{"name":"sum___v","type":"longSum","fieldName":"__v"},{"name":"sum_status_code","type":"longMax","fieldName":"status_code"},{"type":"HLLSketchBuild","name":"count_of_user_id","fieldName":"user_id"},{"type":"HLLSketchBuild","name":"count_of_survey_submission_id","fieldName":"survey_submission_id"},{"type":"HLLSketchBuild","name":"count_of_solution_id","fieldName":"solution_id"},{"type":"HLLSketchBuild","name":"count_of_program_id","fieldName":"program_id"}]}}}' ml_analytics_survey_rollup_output_dir: "/opt/sparkjobs/source/survey/output" ml_analytics_project_rollup_cloud_blob_path: "projects/rollup" ml_analytics_observation_rollup_cloud_blob_path: "observation/rollup" ml_analytics_project_rollup_output_dir: "/opt/sparkjobs/source/projects/output_rollup" ml_analytics_observation_status_rollup_output_dir: "/opt/sparkjobs/source/observations/output_rollup" -ml_analytics_druid_project_rollup_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/rollup/projects_rollup.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"iso"},"dimensionsSpec":{"dimensions":["project_title","project_goal","area_of_improvement","status_of_project","tasks_name","tasks_status","designation","task_evidence_status","project_id","task_id","project_created_type","parent_channel","program_id","program_name","project_updated_date","createdBy","program_externalId","private_program","task_deleted_flag","project_terms_and_condition","state_externalId","block_externalId","district_externalId","cluster_externalId","school_externalId","state_name","block_name","district_name","cluster_name","school_name","board_name","organisation_name","solution_id","organisation_id",{"name":"status_code","type":"long"}]},"metricsSpec":[{"name":"count","type":"count"},{"name":"sum___v","type":"longSum","fieldName":"__v"},{"name":"sum_status_code","type":"longMax","fieldName":"status_code"},{"type":"HLLSketchBuild","name":"count_of_createBy","fieldName":"createdBy"},{"type":"HLLSketchBuild","name":"count_of_project_id","fieldName":"project_id"},{"type":"HLLSketchBuild","name":"count_of_solution_id","fieldName":"solution_id"},{"type":"HLLSketchBuild","name":"count_of_program_id","fieldName":"program_id"}]}}}' -ml_analytics_druid_observation_status_rollup_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/rollup/observation_status_rollup.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-observation-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["status","user_id","solution_id","submission_id","entity_name","completedDate","program_id","private_program","solution_type","updatedAt","role_title","solution_name","program_name","channel","parent_channel","block_name","district_name","school_name","cluster_name","state_name","organisation_name","board_name","district_externalId","state_externalId","block_externalId","cluster_externalId","school_externalId","organisation_id",{"type":"long","name":"status_code"}]},"metricsSpec":[{"type":"count","name":"count"},{"type":"longSum","name":"sum___v","fieldName":"__v","expression":null},{"type":"HLLSketchBuild","name":"count_distinct_solution","fieldName":"solution_id","lgK":12,"tgtHllType":"HLL_4","round":false},{"type":"HLLSketchBuild","name":"count_distinct_submission_id","fieldName":"submission_id","lgK":12,"tgtHllType":"HLL_4","round":false},{"type":"HLLSketchBuild","name":"count_distinct_user_id","fieldName":"user_id","lgK":12,"tgtHllType":"HLL_4","round":false}]}}}' +ml_analytics_druid_project_rollup_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"iso"},"dimensionsSpec":{"dimensions":["project_title","project_goal","area_of_improvement","status_of_project","tasks_name","tasks_status","designation","task_evidence_status","project_id","task_id","project_created_type","parent_channel","program_id","program_name","project_updated_date","createdBy","program_externalId","private_program","task_deleted_flag","project_terms_and_condition","state_externalId","block_externalId","district_externalId","cluster_externalId","school_externalId","state_name","block_name","district_name","cluster_name","school_name","board_name","organisation_name","solution_id","organisation_id",{"name":"status_code","type":"long"}]},"metricsSpec":[{"name":"count","type":"count"},{"name":"sum___v","type":"longSum","fieldName":"__v"},{"name":"sum_status_code","type":"longMax","fieldName":"status_code"},{"type":"HLLSketchBuild","name":"count_of_createBy","fieldName":"createdBy"},{"type":"HLLSketchBuild","name":"count_of_project_id","fieldName":"project_id"},{"type":"HLLSketchBuild","name":"count_of_solution_id","fieldName":"solution_id"},{"type":"HLLSketchBuild","name":"count_of_program_id","fieldName":"program_id"}]}}}' +ml_analytics_druid_observation_status_rollup_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-observation-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["status","user_id","solution_id","submission_id","entity_name","completedDate","program_id","private_program","solution_type","updatedAt","role_title","solution_name","program_name","channel","parent_channel","block_name","district_name","school_name","cluster_name","state_name","organisation_name","board_name","district_externalId","state_externalId","block_externalId","cluster_externalId","school_externalId","organisation_id",{"type":"long","name":"status_code"}]},"metricsSpec":[{"type":"count","name":"count"},{"type":"longSum","name":"sum___v","fieldName":"__v","expression":null},{"type":"HLLSketchBuild","name":"count_distinct_solution","fieldName":"solution_id","lgK":12,"tgtHllType":"HLL_4","round":false},{"type":"HLLSketchBuild","name":"count_distinct_submission_id","fieldName":"submission_id","lgK":12,"tgtHllType":"HLL_4","round":false},{"type":"HLLSketchBuild","name":"count_distinct_user_id","fieldName":"user_id","lgK":12,"tgtHllType":"HLL_4","round":false}]}}}' ml_analytics_druid_rollup_url: "{{groups['druid'][0]}}:8081" -ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/distinctCountPrglevel/ml_projects_distinctCount_prgmlevel.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-programLevel-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"},{"type":"long","name":"no_of_certificate_issued"}]},"metricsSpec":[]}}}' +ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-programLevel-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"},{"type":"long","name":"no_of_certificate_issued"}]},"metricsSpec":[]}}}' ml_analytics_projects_distinctCnt_prglevel_output_dir: "{{ WORKDIR }}/source/projects/distinctCountPrglevel/output" ml_analytics_projects_distinctCnt_prglevel_cloud_blob_path: "projects/distinctCountPrglevel/" ml_analytics_survey_status_output_dir : "{{ WORKDIR }}/source/survey/status/output" ml_analytics_survey_cloud_blob_path: "survey/status/" -ml_analytics_druid_survey_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/survey/status/sl_survey_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel", "solution_name", "solution_id","private_program","state_code","school_code","district_code","block_code","cluster_code"]},"metricsSpec":[]}}}' +ml_analytics_survey_distinctCnt_cloud_blob_path: "survey/distinctCount/" +ml_analytics_druid_survey_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel", "solution_name", "solution_id","private_program","state_code","school_code","district_code","block_code","cluster_code"]},"metricsSpec":[]}}}' ml_slack_token: "{{ ml_analytics_slack_token | default('') }}" ml_slack_channel: "{{ ml_analytics_slack_channel | default('') }}" ml_analytics_program_dashboard_cloud_blob_path: "{{ ml_program_blob_path | default('') }}" @@ -90,12 +93,12 @@ ml_druid_query_data: "{{ ml_druid_query | default('') }}" ml_program_dashboard_data: "{{ ml_program_data | default('') }}" ml_analytics_druid_query_url: "{{groups['druid'][0]}}:8082" ml_analytics_druid_observation_query_spec: '{"queryType":"scan","dataSource":"sl-observation","resultFormat":"list","columns":["completedDate","createdAt","createdBy","criteriaExternalId","criteriaId","criteriaName","entityType","entityTypeId","observationId","observationName","observationSubmissionId","questionAnswer","questionECM","questionExternalId","questionId","questionName","questionResponseLabel","questionResponseType","solutionExternalId","solutionId","solutionName","updatedAt","instanceParentId","instanceId","instanceParentResponsetype","instanceParentQuestion","questionSequenceByEcm","maxScore","minScore","percentageScore","pointsBasedScoreInParent","totalScore","scoreAchieved","totalpercentage","instanceParentExternalId","instanceParentEcmSequence","remarks","total_evidences","evidence_count","school","block","district","cluster","state","schoolName","blockName","districtName","clusterName","stateName","schoolExternalId","blockExternalId","districtExternalId","clusterExternalId","stateExternalId","schoolTypes","administrationTypes","instanceParentCriteriaId","instanceParentCriteriaExternalId","instanceParentCriteriaName","role_title","location_validated_with_geotag","distance_in_meters","entity","entityExternalId","entityName","isAPrivateProgram","programId","programName","programExternalId","questionResponseLabel_number","criteriaLevel","criteriaScore","submissionNumber","submissionTitle","channel","parent_channel","user_districtName","user_blockName","user_clusterName","appName","evidences","user_stateName","domainName","domainExternalId","childName","childType","childExternalid","level","criteriaDescription","programDescription","solutionDescription","label","imp_project_id","imp_project_title","imp_project_goal","imp_project_externalId","ancestorName","scoringSystem","domainLevel","domainScore","criteriaLevelReport","user_schoolName","user_schoolId","user_schoolUDISE_code","solution_type","organisation_name","user_boardName","district_externalId","state_externalId","block_externalId","cluster_externalId","organisation_id","user_type"],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]}' -ml_analytics_druid_observation_batch_ingestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/observation/batchDeletion/druidData.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"entityType"},{"type":"string","name":"entityTypeId"},{"type":"string","name":"observationId"},{"type":"string","name":"observationName"},{"type":"string","name":"observationSubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"school"},{"type":"string","name":"block"},{"type":"string","name":"district"},{"type":"string","name":"cluster"},{"type":"string","name":"state"},{"type":"string","name":"schoolName"},{"type":"string","name":"blockName"},{"type":"string","name":"districtName"},{"type":"string","name":"clusterName"},{"type":"string","name":"stateName"},{"type":"string","name":"schoolExternalId"},{"type":"string","name":"blockExternalId"},{"type":"string","name":"districtExternalId"},{"type":"string","name":"clusterExternalId"},{"type":"string","name":"stateExternalId"},{"type":"string","name":"schoolTypes"},{"type":"string","name":"administrationTypes"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"role_title"},{"type":"string","name":"location_validated_with_geotag"},{"type":"string","name":"distance_in_meters"},{"type":"string","name":"entity"},{"type":"string","name":"entityExternalId"},{"type":"string","name":"entityName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"criteriaLevel"},{"type":"string","name":"criteriaScore"},{"type":"string","name":"submissionNumber"},{"type":"string","name":"submissionTitle"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"user_districtName"},{"type":"string","name":"user_blockName"},{"type":"string","name":"user_clusterName"},{"type":"string","name":"appName"},{"type":"string","name":"evidences"},{"type":"string","name":"user_stateName"},{"type":"string","name":"domainName"},{"type":"string","name":"domainExternalId"},{"type":"string","name":"childName"},{"type":"string","name":"childType"},{"type":"string","name":"childExternalid"},{"type":"string","name":"level"},{"type":"string","name":"criteriaDescription"},{"type":"string","name":"programDescription"},{"type":"string","name":"solutionDescription"},{"type":"string","name":"label"},{"type":"string","name":"imp_project_id"},{"type":"string","name":"imp_project_title"},{"type":"string","name":"imp_project_goal"},{"type":"string","name":"imp_project_externalId"},{"type":"string","name":"ancestorName"},{"type":"string","name":"scoringSystem"},{"type":"string","name":"domainLevel"},{"type":"string","name":"domainScore"},{"name":"criteriaLevelReport","type":"boolean"},{"type":"string","name":"user_schoolName"},{"type":"string","name":"user_schoolId"},{"type":"string","name":"user_schoolUDISE_code"},{"type":"string","name":"solution_type"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_boardName"},{"type":"string","name":"district_externalId"},{"type":"string","name":"state_externalId"},{"type":"string","name":"block_externalId"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"user_type"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}' +ml_analytics_druid_observation_batch_ingestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"entityType"},{"type":"string","name":"entityTypeId"},{"type":"string","name":"observationId"},{"type":"string","name":"observationName"},{"type":"string","name":"observationSubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"school"},{"type":"string","name":"block"},{"type":"string","name":"district"},{"type":"string","name":"cluster"},{"type":"string","name":"state"},{"type":"string","name":"schoolName"},{"type":"string","name":"blockName"},{"type":"string","name":"districtName"},{"type":"string","name":"clusterName"},{"type":"string","name":"stateName"},{"type":"string","name":"schoolExternalId"},{"type":"string","name":"blockExternalId"},{"type":"string","name":"districtExternalId"},{"type":"string","name":"clusterExternalId"},{"type":"string","name":"stateExternalId"},{"type":"string","name":"schoolTypes"},{"type":"string","name":"administrationTypes"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"role_title"},{"type":"string","name":"location_validated_with_geotag"},{"type":"string","name":"distance_in_meters"},{"type":"string","name":"entity"},{"type":"string","name":"entityExternalId"},{"type":"string","name":"entityName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"criteriaLevel"},{"type":"string","name":"criteriaScore"},{"type":"string","name":"submissionNumber"},{"type":"string","name":"submissionTitle"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"user_districtName"},{"type":"string","name":"user_blockName"},{"type":"string","name":"user_clusterName"},{"type":"string","name":"appName"},{"type":"string","name":"evidences"},{"type":"string","name":"user_stateName"},{"type":"string","name":"domainName"},{"type":"string","name":"domainExternalId"},{"type":"string","name":"childName"},{"type":"string","name":"childType"},{"type":"string","name":"childExternalid"},{"type":"string","name":"level"},{"type":"string","name":"criteriaDescription"},{"type":"string","name":"programDescription"},{"type":"string","name":"solutionDescription"},{"type":"string","name":"label"},{"type":"string","name":"imp_project_id"},{"type":"string","name":"imp_project_title"},{"type":"string","name":"imp_project_goal"},{"type":"string","name":"imp_project_externalId"},{"type":"string","name":"ancestorName"},{"type":"string","name":"scoringSystem"},{"type":"string","name":"domainLevel"},{"type":"string","name":"domainScore"},{"name":"criteriaLevelReport","type":"boolean"},{"type":"string","name":"user_schoolName"},{"type":"string","name":"user_schoolId"},{"type":"string","name":"user_schoolUDISE_code"},{"type":"string","name":"solution_type"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_boardName"},{"type":"string","name":"district_externalId"},{"type":"string","name":"state_externalId"},{"type":"string","name":"block_externalId"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"user_type"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}' ml_analytics_observation_batchupdate_cloud_blob_path: "observation/batchDeletion" ml_analytics_observation_submission_id_filepath: "{{ WORKDIR }}/ml-analytics-service/observations/submissions.csv" ml_analytics_observation_batchupdate_output_dir: "{{ WORKDIR }}/source/observations/" ml_analytics_druid_survey_query_spec : '{"queryType":"scan","dataSource":"sl-survey","resultFormat":"list","columns":["completedDate","createdAt","createdBy","criteriaExternalId","criteriaId","criteriaName","surveyId","surveyName","surveySubmissionId","questionAnswer","questionECM","questionExternalId","questionId","questionName","questionResponseLabel","questionResponseType","solutionExternalId","solutionId","solutionName","updatedAt","instanceParentId","instanceId","instanceParentResponsetype","instanceParentQuestion","questionSequenceByEcm","maxScore","minScore","percentageScore","pointsBasedScoreInParent","totalScore","scoreAchieved","totalpercentage","instanceParentExternalId","instanceParentEcmSequence","remarks","total_evidences","evidence_count","instanceParentCriteriaId","instanceParentCriteriaExternalId","instanceParentCriteriaName","isAPrivateProgram","programId","programName","programExternalId","questionResponseLabel_number","channel","parent_channel","appName","organisation_name","user_subtype","user_type","board_name","district_code","district_name","district_externalId","block_code","block_name","block_externalId","school_code","school_name","school_externalId","cluster_code","cluster_name","cluster_externalId","state_code","state_name","state_externalId","organisation_id","evidences"],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]}' -ml_analytics_druid_survey_batch_ingestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris": ["azure://telemetry-data-store/survey/batchDeletion/druidData.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-survey","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"surveyId"},{"type":"string","name":"surveyName"},{"type":"string","name":"surveySubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"evidences"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"appName"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_subtype"},{"type":"string","name":"user_type"},{"type":"string","name":"board_name"},{"type":"string","name":"district_code"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_code"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"school_code"},{"type":"string","name":"school_name"},{"type":"string","name":"school_externalId"},{"type":"string","name":"cluster_code"},{"type":"string","name":"cluster_name"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"state_code"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}' +ml_analytics_druid_survey_batch_ingestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-survey","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"surveyId"},{"type":"string","name":"surveyName"},{"type":"string","name":"surveySubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"evidences"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"appName"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_subtype"},{"type":"string","name":"user_type"},{"type":"string","name":"board_name"},{"type":"string","name":"district_code"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_code"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"school_code"},{"type":"string","name":"school_name"},{"type":"string","name":"school_externalId"},{"type":"string","name":"cluster_code"},{"type":"string","name":"cluster_name"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"state_code"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}' ml_analytics_survey_batchupdate_cloud_blob_path: "survey/batchDeletion" ml_analytics_survey_submission_id_filepath: "{{ WORKDIR }}/ml-analytics-service/survey/submissions.csv" ml_analytics_survey_batchupdate_output_dir: "{{ WORKDIR }}/source/survey/" diff --git a/ansible/roles/ml-analytics-service/templates/config.j2 b/ansible/roles/ml-analytics-service/templates/config.j2 index e15c7eb70b..789a9bc090 100644 --- a/ansible/roles/ml-analytics-service/templates/config.j2 +++ b/ansible/roles/ml-analytics-service/templates/config.j2 @@ -93,6 +93,8 @@ ml_distinctCnt_projects_status_spec = {{ ml_analytics_druid_distinctCnt_projects ml_distinctCnt_prglevel_projects_status_spec = {{ ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec }} +ml_distinctCnt_survey_status_spec ={{ ml_analytics_druid_distinctCnt_survey_injestion_spec }} + observation_status_rollup_injestion_spec = {{ ml_analytics_druid_observation_status_rollup_injestion_spec }} project_rollup_injestion_spec = {{ ml_analytics_druid_project_rollup_injestion_spec }} @@ -209,6 +211,8 @@ projects_distinctCount = {{ ml_analytics_projects_distinctCnt_output_dir }} projects_distinctCount_prgmlevel = {{ ml_analytics_projects_distinctCnt_prglevel_output_dir }} +survey_distinctCount_status = {{ ml_analytics_survey_distinctCnt_output_dir }} + project_rollup = {{ ml_analytics_project_rollup_output_dir }} observation_status_rollup = {{ ml_analytics_observation_status_rollup_output_dir }} @@ -259,6 +263,8 @@ projects_distinctCnt_blob_path = {{ ml_analytics_projects_distinctCnt_cloud_blob projects_distinctCnt_prgmlevel_blob_path = {{ ml_analytics_projects_distinctCnt_prglevel_cloud_blob_path }} +survey_distinctCount_blob_path = {{ ml_analytics_survey_distinctCnt_cloud_blob_path }} + projects_rollup_blob_path = {{ ml_analytics_project_rollup_cloud_blob_path }} observation_rollup_blob_path = {{ ml_analytics_observation_rollup_cloud_blob_path }} From 7220872b7f67a928c4d0b45199754ae63ddb536d Mon Sep 17 00:00:00 2001 From: VISHNUDAS-tunerlabse Date: Wed, 16 Aug 2023 14:53:34 +0530 Subject: [PATCH 41/53] changed to public variables for cloud config --- .../roles/stack-sunbird/templates/ml-core-service.env | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/roles/stack-sunbird/templates/ml-core-service.env b/ansible/roles/stack-sunbird/templates/ml-core-service.env index 30018c5cd8..56313ff44f 100755 --- a/ansible/roles/stack-sunbird/templates/ml-core-service.env +++ b/ansible/roles/stack-sunbird/templates/ml-core-service.env @@ -22,12 +22,12 @@ GOTENBERG_SERVICE_URL={{ml_gotenberg_url | default("http://gotenberg-service:300 # CLOUD STORAGE CLOUD_STORAGE_PROVIDER={{cloud_service_provider}} -CLOUD_STORAGE_ACCOUNTNAME={{cloud_private_storage_accountname | default("")}} -CLOUD_STORAGE_SECRET={{cloud_private_storage_secret | default("")}} +CLOUD_STORAGE_ACCOUNTNAME={{cloud_public_storage_accountname | default("")}} +CLOUD_STORAGE_SECRET={{cloud_public_storage_secret | default("")}} CLOUD_STORAGE_BUCKETNAME={{ml_container | default("")}} -CLOUD_STORAGE_REGION={{cloud_private_storage_region | default("")}} -CLOUD_STORAGE_PROJECT={{cloud_private_storage_project | default("")}} -CLOUD_ENDPOINT={{cloud_private_storage_endpoint | default("")}} +CLOUD_STORAGE_REGION={{cloud_public_storage_region | default("")}} +CLOUD_STORAGE_PROJECT={{cloud_public_storage_project | default("")}} +CLOUD_ENDPOINT={{cloud_public_storage_endpoint | default("")}} # OFFLINE TOKEN VALIDATION From 6d6861e15a46afffb3ae448a50dcf8ad36f79b8e Mon Sep 17 00:00:00 2001 From: adithya_dinesh Date: Mon, 21 Aug 2023 17:50:14 +0530 Subject: [PATCH 42/53] CSP ML Analytics Changes --- .../ml-analytics-service/defaults/main.yml | 1 + .../ml-analytics-service/templates/config.j2 | 49 ++----------------- 2 files changed, 6 insertions(+), 44 deletions(-) diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index 98be525421..15ef432e43 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -10,6 +10,7 @@ ml_analytics_survey_app_name: "{{ ml_survey_app_name | default('sunbirdsurvey') ml_analytics_integrated_app_name: "{{ ml_integrated_app_name | default('sunbird') }}" ml_analytics_integrated_portal: "{{ ml_integrated_portal | default('dev.sunbird.portal') }}" ml_analytics_survey_service: "http://{{private_ingressgateway_ip}}/ml-survey" +ml_analytics_core_service: "http://{{private_ingressgateway_ip}}/ml-core" # SB-31155 # This should be deprecated in future in favour of ml_analytics_public_storage diff --git a/ansible/roles/ml-analytics-service/templates/config.j2 b/ansible/roles/ml-analytics-service/templates/config.j2 index 789a9bc090..935085d36a 100644 --- a/ansible/roles/ml-analytics-service/templates/config.j2 +++ b/ansible/roles/ml-analytics-service/templates/config.j2 @@ -147,54 +147,11 @@ survey_streaming_success_error = {{ ml_analytics_survey_log_folder_path }}/ survey_status_success_error = {{ ml_analytics_survey_log_folder_path }}/status/ -{% if ML_Cloud_Service_Provider is equalto 'ORACLE' %} -[ORACLE] - -endpoint_url = {{ cloud_public_storage_endpoint }} - -access_key = {{ cloud_public_storage_accountname }} - -secret_access_key = {{ cloud_public_storage_secret }} - -region_name = {{ cloud_public_storage_region }} - -bucket_name = {{ cloud_storage_telemetry_bucketname }} - -{% elif ML_Cloud_Service_Provider is equalto 'gcloud' %} - -[GCP] - -secret_data = {{ ml_Cloud_secret_json_file }} - -bucket_name = {{ cloud_storage_telemetry_bucketname }} - -{% elif ML_Cloud_Service_Provider is equalto 'aws' %} - -[AWS] - -service_name = S3 - -access_key = {{ cloud_public_storage_accountname }} - -secret_access_key = {{ cloud_public_storage_secret }} - -region_name = {{ cloud_public_storage_region }} - -bucket_name = {{ cloud_storage_telemetry_bucketname }} - -{% else %} - -[AZURE] - -account_name = {{ cloud_public_storage_accountname }} +[CLOUD] container_name = {{ cloud_storage_telemetry_bucketname }} -account_key = {{ cloud_public_storage_secret }} - -{% endif %} - [OUTPUT_DIR] project = {{ ml_analytics_project_output_dir }} @@ -302,3 +259,7 @@ createdBy = {{ ml_analytics_createdBy }} container = {{ ml_analytics_reports_container }} store = {{ ml_analytics_reports_store }} + +[ML_CORE_SERVICE_URL] + +url = {{ ml_analytics_core_service }} From af22c6997a461e54a25a27d945e501a3c46532e1 Mon Sep 17 00:00:00 2001 From: santhosh-tg <93243580+santhosh-tg@users.noreply.github.com> Date: Tue, 22 Aug 2023 13:08:47 +0530 Subject: [PATCH 43/53] =?UTF-8?q?ED-1251=20-=20Add=20OCI=20specific=20devo?= =?UTF-8?q?ps=20changes=20-=20superseeds=20PR=203817=20,=203735=E2=80=A6?= =?UTF-8?q?=20(#3881)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ED-1251 - Add OCI specific devops changes - superseeds PR 3817 , 3735 (#3858) * added task and role for artifacts upload - oci oss Signed-off-by: Deepak Devadathan * overwrite oss file if exists Signed-off-by: Deepak Devadathan * added the switch for getting oci deployed image tag Signed-off-by: Deepak Devadathan * added oci in the task name for getting image version Signed-off-by: Deepak Devadathan * added helm values for oci for internal lb provisioning Signed-off-by: Deepak Devadathan * getting image tag for oci Signed-off-by: Deepak Devadathan * added oci oss upload for certificate template Signed-off-by: Deepak Devadathan * corrected typo in the role input variable Signed-off-by: Deepak Devadathan * removed prefix flag from bulk upload Signed-off-by: Deepak Devadathan * added oci oss task for chatbot upload Signed-off-by: Deepak Devadathan * updated the prefix logic for folder upload to oss Signed-off-by: Deepak Devadathan * uploadfaq to oci oss Signed-off-by: Deepak Devadathan * added a temp debug Signed-off-by: Deepak Devadathan * testing bulk upload logic Signed-off-by: Deepak Devadathan * testing oss_path variable Signed-off-by: Deepak Devadathan * testing with a defined prefix Signed-off-by: Deepak Devadathan * removed debug line and oss_path value Signed-off-by: Deepak Devadathan * added overwrite flag for bulk upload Signed-off-by: Deepak Devadathan * oci oss changes for uploadschema Signed-off-by: Deepak Devadathan * typo in the csp switch Signed-off-by: Deepak Devadathan * oci oss changes for plugins upload Signed-off-by: Deepak Devadathan * testing by adding privileged: true to init container Signed-off-by: Deepak Devadathan * removed privileged=true..as it didnt help the deployment Signed-off-by: Deepak Devadathan * deployed image check for oci Signed-off-by: Deepak Devadathan * added the switch for selecting api version Signed-off-by: Deepak Devadathan * corrected the right task file for plugin del and upl for oci Signed-off-by: Deepak Devadathan * added a temporary debug Signed-off-by: Deepak Devadathan * testing flow Signed-off-by: Deepak Devadathan * debug line Signed-off-by: Deepak Devadathan * testing Signed-off-by: Deepak Devadathan * disable delete and repload plugin temp Signed-off-by: Deepak Devadathan * testing Signed-off-by: Deepak Devadathan * adding OCI values for player environments * updated daemonset deployed image retrieval Signed-off-by: Deepak Devadathan * added role for oci-cli Signed-off-by: Deepak Devadathan * test home location Signed-off-by: Deepak Devadathan * updated the oci cli install location Signed-off-by: Deepak Devadathan * include option oci-cli in template edit jenkins pipeline template to include oci_cli option * updated oci related task for desktop faq upload Signed-off-by: Deepak Devadathan * added the missing uploads for desktop faq upload Signed-off-by: Deepak Devadathan * added CLOUD_STORAGE_ENDPOINT env variable Signed-off-by: Deepak Devadathan * updated the ansible variable for cloud_storage_endpoint Signed-off-by: Deepak Devadathan * updated env template for lms to add endpoint Signed-off-by: Deepak Devadathan * corrected the typo in the template for lms Signed-off-by: Deepak Devadathan * updated desktop-faq-upload for oci Signed-off-by: Deepak Devadathan * added uploadfaq for oci rc-schema Signed-off-by: Deepak Devadathan * updated bucket name for oci upload for schema Signed-off-by: Deepak Devadathan * updated bucket name for artifact upl and download for oci Signed-off-by: Deepak Devadathan * added oci oss block for asset upload Signed-off-by: Deepak Devadathan * updated oci buket name for deploy plugins Signed-off-by: Deepak Devadathan * added es snapshot for oci Signed-off-by: Deepak Devadathan * added es repository for oci Signed-off-by: Deepak Devadathan * added repository changes for log-es6 for oci Signed-off-by: Deepak Devadathan * removed access key and secret from es s3 repo settings Signed-off-by: Deepak Devadathan * add s3 compatible repository setting for es backup Signed-off-by: Deepak Devadathan * changed the repository type to s3 Signed-off-by: Deepak Devadathan * plugin repository-s3 for oci Signed-off-by: Deepak Devadathan * temporarily disabling no_log for trigage Signed-off-by: Deepak Devadathan * used the logical or Signed-off-by: Deepak Devadathan * reverted no_log Signed-off-by: Deepak Devadathan * default client name based setting Signed-off-by: Deepak Devadathan * added signer override Signed-off-by: Deepak Devadathan * added s3 client configuration for oci Signed-off-by: Deepak Devadathan * corrected elasticsearch.yml template Signed-off-by: Deepak Devadathan * corrected the variable names in elasticsearch.yml Signed-off-by: Deepak Devadathan * removed s3 compatiblity variables from elasticsearch.yaml Signed-off-by: Deepak Devadathan * tsting snapshot backup Signed-off-by: Deepak Devadathan * without path-style-acess Signed-off-by: Deepak Devadathan * testing with home region Signed-off-by: Deepak Devadathan * using ip of endpoint Signed-off-by: Deepak Devadathan * install repository-s3 plugin for log-es Signed-off-by: Deepak Devadathan * updated s3 settings values Signed-off-by: Deepak Devadathan * re arranged the order Signed-off-by: Deepak Devadathan * corrected the default values for sunbird_player.env Signed-off-by: Deepak Devadathan * removed the redundant check to get deployed image Signed-off-by: Deepak Devadathan * removed csp switch for deployed image retrival Signed-off-by: Deepak Devadathan * common login for getting deployed image Signed-off-by: Deepak Devadathan * removed csp switch for getting deployed image Signed-off-by: Deepak Devadathan * added the environment variable for instance principal for oci operation Signed-off-by: Deepak Devadathan * Fixed indentation error * Update oss-upload-batch-no-poll.yml missing content type caused a LOT of problem * Changes for postgres ssl configuration * added the storage type for deployment Signed-off-by: Deepak Devadathan * added cloud-store-proxy for cors reverse proxy Signed-off-by: Deepak Devadathan * enabling relative base path in content API * enabling relative path in content * setting relative absolute path to true * issue with npm version * Fix for DU-417 * Moved the storage config to ansible variables * typo fix * updated oci oss config entries * updated defaults Signed-off-by: Deepak Devadathan * corrected values Signed-off-by: Deepak Devadathan * updated kafka setup zookeeper Signed-off-by: Deepak Devadathan * updated the generic defaults Signed-off-by: Deepak Devadathan * added conditional check for pg ssl Signed-off-by: Deepak Devadathan * corrected misplaced oci instead of aws Signed-off-by: Deepak Devadathan * removed https Signed-off-by: Deepak Devadathan * used variable for cloud_storage_url Signed-off-by: Deepak Devadathan * removed duplicate entry Signed-off-by: Deepak Devadathan * added default values for cloud_public_storage_proxy Signed-off-by: Deepak Devadathan * added defaults to stack sunbird role Signed-off-by: Deepak Devadathan * added genereic names Signed-off-by: Deepak Devadathan * removed redundant file Signed-off-by: Deepak Devadathan * removed redundant variable cloud_store_base_path_placeholder Signed-off-by: Deepak Devadathan * removed redundant variable Signed-off-by: Deepak Devadathan * added default value for cloud_public_storage_proxy Signed-off-by: Deepak Devadathan * added sample reference value for cloud_public_storage_endpoint Signed-off-by: Deepak Devadathan --------- Signed-off-by: Deepak Devadathan Co-authored-by: Kenneth Heung Co-authored-by: subhash_chandra_budde Co-authored-by: Ramya Co-authored-by: rjanart <123344037+rjanart@users.noreply.github.com> * Update sunbird_lms-service.env * Resolving conflicts at sunbird_lms-service.env --------- Signed-off-by: Deepak Devadathan Co-authored-by: Deepak Devadathan Co-authored-by: Kenneth Heung Co-authored-by: subhash_chandra_budde Co-authored-by: Ramya Co-authored-by: rjanart <123344037+rjanart@users.noreply.github.com> Co-authored-by: Santhosh Gandham --- ansible/artifacts-download.yml | 13 +++ ansible/artifacts-upload.yml | 13 +++ ansible/assets-upload.yml | 28 +++++++ ansible/deploy-plugins.yml | 79 +++++++++++++++++- ansible/desktop-faq-upload.yml | 82 ++++++++++++++++++- ansible/es.yml | 4 + ansible/inventory/env/group_vars/all.yml | 6 +- ansible/kp_upload-schema.yml | 13 +++ ansible/roles/cert-templates/tasks/main.yml | 12 +++ .../roles/es-oci-snapshot/defaults/main.yml | 19 +++++ ansible/roles/es-oci-snapshot/tasks/main.yml | 42 ++++++++++ ansible/roles/es6/tasks/main.yml | 2 +- .../roles/es6/templates/elasticsearch.yml.j2 | 1 + ansible/roles/log-es6/tasks/main.yml | 6 +- .../log-es6/tasks/plugins/repository-oci.yml | 36 ++++++++ .../roles/oci-cloud-storage/defaults/main.yml | 3 + .../oci-cloud-storage/tasks/delete-folder.yml | 5 ++ .../roles/oci-cloud-storage/tasks/delete.yml | 7 ++ .../oci-cloud-storage/tasks/download.yml | 7 ++ .../roles/oci-cloud-storage/tasks/main.yml | 18 ++++ .../tasks/oss-delete-batch-no-poll.yml | 5 ++ .../tasks/oss-upload-batch-no-poll.yml | 5 ++ .../oci-cloud-storage/tasks/upload-folder.yml | 15 ++++ .../roles/oci-cloud-storage/tasks/upload.yml | 8 ++ ansible/roles/setup-kafka/defaults/main.yml | 2 +- ansible/roles/setup-kafka/tasks/main.yml | 10 +-- ansible/roles/stack-sunbird/defaults/main.yml | 4 + .../templates/analytics_api_service.conf | 13 +-- .../content-service_application.conf | 1 + ansible/roles/stack-sunbird/templates/odk.env | 8 +- .../templates/sunbird_cert-service.env | 1 + .../templates/sunbird_lms-service.env | 4 +- .../templates/sunbird_player.env | 1 + ansible/uploadFAQs.yml | 30 +++++++ .../roles/deploy-player/tasks/main.yml | 1 + .../ansible/roles/helm-deploy/tasks/main.yml | 1 + .../templates/configmap.yaml | 9 +- .../core/nginx-private-ingress/values.j2 | 1 + .../fluent-bit/templates/serviceaccount.yaml | 8 ++ pipelines/deploy/ContentFramework/Jenkinsfile | 1 + .../ansible/inventory/dev/Core/common.yml | 9 +- .../inventory/dev/DataPipeline/common.yml | 7 +- .../dev/KnowledgePlatform/common.yml | 9 +- 43 files changed, 522 insertions(+), 27 deletions(-) create mode 100644 ansible/roles/es-oci-snapshot/defaults/main.yml create mode 100644 ansible/roles/es-oci-snapshot/tasks/main.yml create mode 100644 ansible/roles/log-es6/tasks/plugins/repository-oci.yml create mode 100644 ansible/roles/oci-cloud-storage/defaults/main.yml create mode 100644 ansible/roles/oci-cloud-storage/tasks/delete-folder.yml create mode 100644 ansible/roles/oci-cloud-storage/tasks/delete.yml create mode 100644 ansible/roles/oci-cloud-storage/tasks/download.yml create mode 100644 ansible/roles/oci-cloud-storage/tasks/main.yml create mode 100644 ansible/roles/oci-cloud-storage/tasks/oss-delete-batch-no-poll.yml create mode 100644 ansible/roles/oci-cloud-storage/tasks/oss-upload-batch-no-poll.yml create mode 100644 ansible/roles/oci-cloud-storage/tasks/upload-folder.yml create mode 100644 ansible/roles/oci-cloud-storage/tasks/upload.yml diff --git a/ansible/artifacts-download.yml b/ansible/artifacts-download.yml index 2fc2748229..ce7a125bde 100644 --- a/ansible/artifacts-download.yml +++ b/ansible/artifacts-download.yml @@ -40,3 +40,16 @@ aws_access_key_id: "{{ cloud_artifact_storage_accountname }}" aws_secret_access_key: "{{ cloud_artifact_storage_secret }}" when: cloud_service_provider == "aws" + + - name: download artifact from oci oss + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: download.yml + vars: + local_file_or_folder_path: "{{ artifact_path }}" + oss_bucket_name: "{{ cloud_storage_artifacts_bucketname }}" + oss_path: "{{ artifact }}" + when: cloud_service_provider == "oci" \ No newline at end of file diff --git a/ansible/artifacts-upload.yml b/ansible/artifacts-upload.yml index 305492afc2..3e444ef163 100644 --- a/ansible/artifacts-upload.yml +++ b/ansible/artifacts-upload.yml @@ -41,3 +41,16 @@ aws_access_key_id: "{{ cloud_artifact_storage_accountname }}" aws_secret_access_key: "{{ cloud_artifact_storage_secret }}" when: cloud_service_provider == "aws" + + - name: upload artifact to oci oss + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: upload.yml + vars: + local_file_or_folder_path: "{{ artifact_path }}" + oss_bucket_name: "{{ cloud_storage_artifacts_bucketname }}" + oss_path: "{{ artifact }}" + when: cloud_service_provider == "oci" \ No newline at end of file diff --git a/ansible/assets-upload.yml b/ansible/assets-upload.yml index b8d5836cc6..b39f1a4694 100644 --- a/ansible/assets-upload.yml +++ b/ansible/assets-upload.yml @@ -75,3 +75,31 @@ vars: local_file_or_folder_path: "{{ assets }}/*" when: cloud_service_provider == "gcloud" + +##### OCI + - name: this block consists of tasks related to oci oss + block: + - name: set common oci variables + set_fact: + oss_bucket_name: "{{ cloud_storage_playercdn_bucketname }}" + oss_path: "" + oss_namespace: "{{oci_namespace}}" + + - name: delete files and folders from oci oss + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: delete-folder.yml + + - name: upload batch of files to oci oss + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: upload-folder.yml + vars: + local_file_or_folder_path: "{{ assets }}" + when: cloud_service_provider == "oci" \ No newline at end of file diff --git a/ansible/deploy-plugins.yml b/ansible/deploy-plugins.yml index 5774a12454..a96af44f5a 100644 --- a/ansible/deploy-plugins.yml +++ b/ansible/deploy-plugins.yml @@ -62,7 +62,6 @@ local_file_or_folder_path: "{{ source_file_name }}" tags: - preview - - block: - name: delete and re-upload plugins include_role: @@ -222,3 +221,81 @@ tags: - plugins when: cloud_service_provider == "aws" + + +################################### OCI tasks ######################### + - name: this block consists of tasks related to oci oss + block: + - name: set common oci variables + set_fact: + oss_bucket_name: "{{ cloud_storage_content_bucketname }}" + oss_namespace: "{{ oci_namespace }}" + tags: + - always + + - block: + - name: delete files and folders from oci oss + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: delete-folder.yml + vars: + oss_path: "{{ folder_name }}/" + tags: + - content-editor + - collection-editor + - generic-editor + - preview + + - block: + - name: upload folder to oci oss + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: upload-folder.yml + vars: + oss_path: "{{ folder_name }}/" + local_file_or_folder_path: "{{ source_name }}" + tags: + - content-editor + - collection-editor + - generic-editor + - preview + - editor + - core-plugins + + - block: + - name: upload file to oci oss + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: upload.yml + vars: + oss_path: "artefacts/content-player/content-player-{{ player_version_number }}.zip" + local_file_or_folder_path: "{{ source_file_name }}" + tags: + - preview + + - block: + - name: delete and re-upload plugins for oci + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: "{{ item[0] }}" + vars: + object_prefix: "content-plugins/{{ item[1] }}/" + local_file_or_folder_path: "{{ source_folder }}/{{ item[1] }}" + with_nested: + - ['oss-delete-batch-no-poll.yml', 'oss-upload-batch-no-poll.yml'] + - "{{ lookup('file', plugins_to_delete_and_upload).split('\n') }}" + tags: + - plugins + when: cloud_service_provider == "oci" diff --git a/ansible/desktop-faq-upload.yml b/ansible/desktop-faq-upload.yml index a95e8828e7..77d56dbc5b 100644 --- a/ansible/desktop-faq-upload.yml +++ b/ansible/desktop-faq-upload.yml @@ -211,7 +211,7 @@ - upload-csv-template - block: - - name: upload folder to aws s3 - discussion-ui + - name: upload folder to aws s3 - discusion-ui include_role: name: aws-cloud-storage tasks_from: upload-folder.yml @@ -222,3 +222,83 @@ tags: - upload-discussion-ui when: cloud_service_provider == "aws" + + +##### OCI Tasks #### + + - name: this block consists of tasks related to oci oss + block: + - name: set common oci variables + set_fact: + local_file_or_folder_path: "{{ playbook_dir }}/../{{ src_file_path }}" + tags: + - always + + - block: + - name: upload file to oci oss public bucket + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: upload.yml + vars: + oss_bucket_name: "{{ cloud_storage_public_bucketname }}" + tags: + - upload-desktop-faq + + - block: + - name: upload file to oci oss private bucket + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: upload.yml + vars: + oss_bucket_name: "{{ cloud_storage_label_bucketname }}" + tags: + - upload-label + + - block: + - name: upload folder to oci oss - chatbot + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: upload-folder.yml + vars: + oss_path: "{{ destination_path }}/" + oss_bucket_name: "{{ cloud_storage_chatbot_bucketname }}" + tags: + - upload-chatbot-config + + - block: + - name: upload folder to oci oss - csv-template + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: upload-folder.yml + vars: + oss_path: "{{ destination_path }}/" + oss_bucket_name: "{{ cloud_storage_sourcing_bucketname }}" + tags: + - upload-csv-template + + - block: + - name: upload folder to oci oss - discussion-ui + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: upload-folder.yml + vars: + oss_path: "{{ destination_path }}/" + oss_bucket_name: "{{ cloud_storage_discussionui_bucketname }}" + tags: + - upload-discussion-ui + when: cloud_service_provider == "oci" diff --git a/ansible/es.yml b/ansible/es.yml index c526414f47..193f81ab27 100644 --- a/ansible/es.yml +++ b/ansible/es.yml @@ -12,6 +12,8 @@ when: cloud_service_provider == "aws" - role: es-gcs-snapshot when: cloud_service_provider == "gcloud" + - role: es-oci-snapshot + when: cloud_service_provider == "oci" - role: es5-snapshot-purge tags: - es_backup @@ -31,6 +33,8 @@ when: cloud_service_provider == "aws" - role: es-gcs-snapshot when: cloud_service_provider == "gcloud" + - role: es-oci-snapshot + when: cloud_service_provider == "oci" - es5-snapshot-purge tags: - log_es_backup diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml index b0b23d8ecf..ab39bf76ae 100644 --- a/ansible/inventory/env/group_vars/all.yml +++ b/ansible/inventory/env/group_vars/all.yml @@ -361,7 +361,7 @@ content_service_blacklisted_resourcetype: '' content_service_whitelisted_resourcetype: '' content_service_whitelisted_mimetype: '' content_service_blacklisted_mimetype: '' -sunbird_cloud_storage_urls: 'https://s3.ap-south-1.amazonaws.com/ekstep-public-{{ekstep_s3_env}}/,https://ekstep-public-{{ekstep_s3_env}}.s3-ap-south-1.amazonaws.com/,{{ upstream_url }}/' +sunbird_cloud_storage_urls: 'https://s3.ap-south-1.amazonaws.com/ekstep-public-{{ekstep_s3_env}}/,https://ekstep-public-{{ekstep_s3_env}}.s3-ap-south-1.amazonaws.com/,{{ upstream_url }}/,https://files.xyz.in/contentsbucket/,https://obj.xyz.in/contentsbucket/' sunbird_email_max_recipients_limit: 100 sunbird_cassandra_consistency_level: local_quorum sunbird_cassandra_replication_strategy: '{"class":"SimpleStrategy","replication_factor":"1"}' @@ -566,3 +566,7 @@ plugin_upstream_url: "{{ upstream_url }}" kp_schema_base_path: "{{ upstream_url }}/schemas/local" # SB-31155 - Moved to the installation public container for now (same place where keycloaka and java artifacts are stored) h5p_library_path: "https://sunbirdpublic.blob.core.windows.net/installation/h5p-standalone-1.3.4.zip" + + +## Added default values +cloud_public_storage_proxy: "{{cloud_storage_url}}" \ No newline at end of file diff --git a/ansible/kp_upload-schema.yml b/ansible/kp_upload-schema.yml index 3a28ce5782..0f57586047 100644 --- a/ansible/kp_upload-schema.yml +++ b/ansible/kp_upload-schema.yml @@ -40,3 +40,16 @@ gcp_path: "schemas/local" local_file_or_folder_path: "{{ source_name }}" when: cloud_service_provider == "gcloud" + + - name: upload batch of files to oci oss + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: upload-folder.yml + vars: + oss_bucket_name: "{{ cloud_storage_content_bucketname }}" + local_file_or_folder_path: "{{ source_name }}" + oss_path: "schemas/local/" + when: cloud_service_provider == "oci" \ No newline at end of file diff --git a/ansible/roles/cert-templates/tasks/main.yml b/ansible/roles/cert-templates/tasks/main.yml index 893e0776f4..d4134d7cbd 100644 --- a/ansible/roles/cert-templates/tasks/main.yml +++ b/ansible/roles/cert-templates/tasks/main.yml @@ -68,3 +68,15 @@ gcp_path: "" local_file_or_folder_path: "{{ cert_location }}/cert-templates/certUtilScripts/out" when: cloud_service_provider == "gcloud" + +- name: upload batch of files to oci oss + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: upload-folder.yml + vars: + oss_bucket_name: "{{ cert_service_container_name }}" + local_file_or_folder_path: "{{ cert_location }}/cert-templates/certUtilScripts/out" + when: cloud_service_provider == "oci" \ No newline at end of file diff --git a/ansible/roles/es-oci-snapshot/defaults/main.yml b/ansible/roles/es-oci-snapshot/defaults/main.yml new file mode 100644 index 0000000000..a172208ded --- /dev/null +++ b/ansible/roles/es-oci-snapshot/defaults/main.yml @@ -0,0 +1,19 @@ +snapshot_create_request_body: { + type: s3, + settings: { + bucket: "{{ cloud_storage_esbackup_bucketname }}", + path_style_access: "{{s3_path_style_access}}", + endpoint: "{{s3_storage_endpoint}}", + region: "{{s3_region}}", + base_path: "{{ cloud_storage_esbackup_foldername }}/{{ snapshot_base_path }}_{{ base_path_date }}", + + } +} + +# Override these values +es_snapshot_host: "localhost" +snapshot_base_path: "default" +es_backup_storage: "elasticsearch-snapshots" + +cloud_storage_esbackup_bucketname: "{{ cloud_storage_management_bucketname }}" +cloud_storage_esbackup_foldername: "elasticsearch-snapshots" diff --git a/ansible/roles/es-oci-snapshot/tasks/main.yml b/ansible/roles/es-oci-snapshot/tasks/main.yml new file mode 100644 index 0000000000..aee768626c --- /dev/null +++ b/ansible/roles/es-oci-snapshot/tasks/main.yml @@ -0,0 +1,42 @@ +--- + +- set_fact: base_path_date="{{ lookup('pipe','date +%Y-%m') }}" + +- set_fact: snapshot_number="snapshot_{{ lookup('pipe','date +%s') }}" + +- name: Create S3 Repository + uri: + url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}" + method: PUT + body: "{{ snapshot_create_request_body | to_json }}" + headers: + Content-Type: "application/json" + +- name: Take new snapshot + uri: + url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}" + method: PUT + headers: + Content-Type: "application/json" + +- name: Print all snapshots + uri: + url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/_all" + method: GET + +- name: Print status of current snapshot + uri: + url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}" + method: GET + +- name: "Wait for backup to be completed" + uri: + url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}" + method: GET + return_content: yes + status_code: 200 + body_format: json + register: result + until: result.json.snapshots[0].state == 'SUCCESS' + retries: 120 + delay: 10 diff --git a/ansible/roles/es6/tasks/main.yml b/ansible/roles/es6/tasks/main.yml index 1de1eeff8e..39f915a018 100644 --- a/ansible/roles/es6/tasks/main.yml +++ b/ansible/roles/es6/tasks/main.yml @@ -46,7 +46,7 @@ - name: include plugins/repository-s3.yml include: plugins/repository-s3.yml - when: cloud_service_provider == "aws" + when: cloud_service_provider == "aws" or cloud_service_provider == "oci" - name: include plugins/repository-gcs.yml include: plugins/repository-gcs.yml diff --git a/ansible/roles/es6/templates/elasticsearch.yml.j2 b/ansible/roles/es6/templates/elasticsearch.yml.j2 index d6aaab45c9..880c27da2b 100644 --- a/ansible/roles/es6/templates/elasticsearch.yml.j2 +++ b/ansible/roles/es6/templates/elasticsearch.yml.j2 @@ -56,3 +56,4 @@ network.host: 0.0.0.0 {% if es_remote_reindex is defined %} reindex.remote.whitelist: {{es_remote_host}}:9200 {% endif %} + diff --git a/ansible/roles/log-es6/tasks/main.yml b/ansible/roles/log-es6/tasks/main.yml index 646947abee..8ac712117c 100644 --- a/ansible/roles/log-es6/tasks/main.yml +++ b/ansible/roles/log-es6/tasks/main.yml @@ -68,12 +68,16 @@ - name: include plugins/repository-s3.yml include: plugins/repository-s3.yml - when: cloud_service_provider == "aws" + when: cloud_service_provider == "aws" or cloud_service_provider == "oci" - name: include plugins/repository-gcs.yml include: plugins/repository-gcs.yml when: cloud_service_provider == "gcloud" +- name: include plugins/repository-oci.yml + include: plugins/repository-oci.yml + when: cloud_service_provider == "oci" + - name: include elasticsearch-ssl.yml include: elasticsearch-ssl.yml when: es_enable_http_ssl or es_enable_transport_ssl diff --git a/ansible/roles/log-es6/tasks/plugins/repository-oci.yml b/ansible/roles/log-es6/tasks/plugins/repository-oci.yml new file mode 100644 index 0000000000..02a2b45f45 --- /dev/null +++ b/ansible/roles/log-es6/tasks/plugins/repository-oci.yml @@ -0,0 +1,36 @@ +--- +- name: Add default oci account name for backups + become: yes + shell: echo "{{ cloud_management_storage_accountname }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.access_key + no_log: True + environment: + ES_PATH_CONF: "{{ conf_dir }}" + +- name: Add default oci account key for backups + become: yes + shell: echo "{{ cloud_management_storage_secret }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.secret_key + no_log: True + environment: + ES_PATH_CONF: "{{ conf_dir }}" + + +- name: Add default oci endpoint for backups + become: yes + shell: echo "{{ cloud_management_storage_endpoint }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.endpoint + no_log: True + environment: + ES_PATH_CONF: "{{ conf_dir }}" + +- name: Add default oci region for backups + become: yes + shell: echo "{{ cloud_management_storage_region }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.region + no_log: True + environment: + ES_PATH_CONF: "{{ conf_dir }}" + +- name: Add default path-style access for backups + become: yes + shell: echo "{{ cloud_management_storage_path_style_access }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.path_style_access + no_log: True + environment: + ES_PATH_CONF: "{{ conf_dir }}" \ No newline at end of file diff --git a/ansible/roles/oci-cloud-storage/defaults/main.yml b/ansible/roles/oci-cloud-storage/defaults/main.yml new file mode 100644 index 0000000000..72727de167 --- /dev/null +++ b/ansible/roles/oci-cloud-storage/defaults/main.yml @@ -0,0 +1,3 @@ +oss_bucket_name: "" +oss_path: "" +local_file_or_folder_path: "" diff --git a/ansible/roles/oci-cloud-storage/tasks/delete-folder.yml b/ansible/roles/oci-cloud-storage/tasks/delete-folder.yml new file mode 100644 index 0000000000..6ed4e6b8b4 --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/delete-folder.yml @@ -0,0 +1,5 @@ +--- +- name: delete files and folders recursively + shell: "oci os object bulk-delete -ns {{oss_namespace}} -bn {{oss_bucket_name}} --prefix {{oss_path}} --force" + async: 3600 + poll: 10 diff --git a/ansible/roles/oci-cloud-storage/tasks/delete.yml b/ansible/roles/oci-cloud-storage/tasks/delete.yml new file mode 100644 index 0000000000..65d18843ca --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/delete.yml @@ -0,0 +1,7 @@ +- name: Ensure oci oss bucket exists + command: oci os bucket get --name {{ oss_bucket_name }} + +- name: Upload to oci oss bucket + command: oci os object delete -bn {{ oss_bucket_name }} --name {{ oss_path }} --force + async: 3600 + poll: 10 \ No newline at end of file diff --git a/ansible/roles/oci-cloud-storage/tasks/download.yml b/ansible/roles/oci-cloud-storage/tasks/download.yml new file mode 100644 index 0000000000..63e776c348 --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/download.yml @@ -0,0 +1,7 @@ +- name: Ensure oci oss bucket exists + command: oci os bucket get --name {{ oss_bucket_name }} + +- name: download files from oci oss bucket + command: oci os object bulk-download -bn {{ oss_bucket_name }} --prefix {{ oss_path }} --dest-dir {{ local_file_or_folder_path }} + async: 3600 + poll: 10 \ No newline at end of file diff --git a/ansible/roles/oci-cloud-storage/tasks/main.yml b/ansible/roles/oci-cloud-storage/tasks/main.yml new file mode 100644 index 0000000000..6f9dca6b63 --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/main.yml @@ -0,0 +1,18 @@ +--- +- name: delete files from oci oss bucket + include: delete.yml + +- name: delete folders from oci oss bucket recursively + include: delete-folder.yml + + +- name: download file from oss + include: download.yml + +- name: upload files from a local to oci oss + include: upload.yml + +- name: upload files and folder from local directory to oci oss + include: upload-folder.yml + + diff --git a/ansible/roles/oci-cloud-storage/tasks/oss-delete-batch-no-poll.yml b/ansible/roles/oci-cloud-storage/tasks/oss-delete-batch-no-poll.yml new file mode 100644 index 0000000000..8ad3f257e3 --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/oss-delete-batch-no-poll.yml @@ -0,0 +1,5 @@ +--- +- name: delete files and folders - deleting {{ oss_bucket_name }}/{{ object_prefix }} + shell: oci os object bulk-delete -bn {{oss_bucket_name}} --prefix {{object_prefix}} --force + async: 1800 + poll: 0 \ No newline at end of file diff --git a/ansible/roles/oci-cloud-storage/tasks/oss-upload-batch-no-poll.yml b/ansible/roles/oci-cloud-storage/tasks/oss-upload-batch-no-poll.yml new file mode 100644 index 0000000000..31794e73e8 --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/oss-upload-batch-no-poll.yml @@ -0,0 +1,5 @@ +--- +- name: upload files and folders - uploading {{ oss_bucket_name }}/{{ object_prefix }} + shell: oci os object bulk-upload -bn {{oss_bucket_name}} --prefix {{object_prefix}} --src-dir {{local_file_or_folder_path}} --overwrite --content-type auto + async: 1800 + poll: 0 diff --git a/ansible/roles/oci-cloud-storage/tasks/upload-folder.yml b/ansible/roles/oci-cloud-storage/tasks/upload-folder.yml new file mode 100644 index 0000000000..2e0d45bcb7 --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/upload-folder.yml @@ -0,0 +1,15 @@ +--- +- name: Ensure oci oss bucket exists + command: oci os bucket get --name {{ oss_bucket_name }} + +- name: Upload folder to oci oss bucket + command: oci os object bulk-upload -bn {{ oss_bucket_name }} --src-dir {{ local_file_or_folder_path }} --content-type auto --overwrite + async: 3600 + poll: 10 + when: oss_path|length == 0 + +- name: Upload folder to oci oss bucket + command: oci os object bulk-upload -bn {{ oss_bucket_name }} --prefix {{oss_path}} --src-dir {{ local_file_or_folder_path }} --content-type auto --overwrite + async: 3600 + poll: 10 + when: oss_path|length > 0 diff --git a/ansible/roles/oci-cloud-storage/tasks/upload.yml b/ansible/roles/oci-cloud-storage/tasks/upload.yml new file mode 100644 index 0000000000..9e1ceb4289 --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/upload.yml @@ -0,0 +1,8 @@ +--- +- name: Ensure oci oss bucket exists + command: oci os bucket get --name {{ oss_bucket_name }} + +- name: Upload to oci oss bucket + command: oci os object put -bn {{ oss_bucket_name }} --name {{ oss_path }} --file {{ local_file_or_folder_path }} --content-type auto --force + async: 3600 + poll: 10 diff --git a/ansible/roles/setup-kafka/defaults/main.yml b/ansible/roles/setup-kafka/defaults/main.yml index 293a9f9988..fe276ab941 100644 --- a/ansible/roles/setup-kafka/defaults/main.yml +++ b/ansible/roles/setup-kafka/defaults/main.yml @@ -3,7 +3,7 @@ env: dev ml_kafka_topic_create: false uci_kafka_topic_create: false av_kafka_topic_create: false - +zookeeper_host: "{{ groups['zookeeper'][0] | default('localhost')}}" processing_kafka_overriden_topics: - name: lms.audit.events retention_time: 172800000 diff --git a/ansible/roles/setup-kafka/tasks/main.yml b/ansible/roles/setup-kafka/tasks/main.yml index 502c627bdc..28ef75f422 100644 --- a/ansible/roles/setup-kafka/tasks/main.yml +++ b/ansible/roles/setup-kafka/tasks/main.yml @@ -1,5 +1,5 @@ - name: create topics - command: /opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic {{env_name}}.{{item.name}} --partitions {{ item.num_of_partitions }} --replication-factor {{ item.replication_factor }} + command: /opt/kafka/bin/kafka-topics.sh --zookeeper {{zookeeper_host}}:2181 --create --topic {{env_name}}.{{item.name}} --partitions {{ item.num_of_partitions }} --replication-factor {{ item.replication_factor }} with_items: "{{processing_kafka_topics}}" ignore_errors: true when: kafka_id=="1" @@ -7,14 +7,14 @@ - processing-kafka - name: override retention time - command: /opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --alter --topic {{env_name}}.{{item.name}} --config retention.ms={{ item.retention_time }} + command: /opt/kafka/bin/kafka-topics.sh --zookeeper {{zookeeper_host}}:2181 --alter --topic {{env_name}}.{{item.name}} --config retention.ms={{ item.retention_time }} with_items: "{{processing_kafka_overriden_topics}}" when: kafka_id=="1" and item.retention_time is defined tags: - processing-kafka - name: create topics - command: /opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic {{env_name}}.{{item.name}} --partitions {{ item.num_of_partitions }} --replication-factor {{ item.replication_factor }} + command: /opt/kafka/bin/kafka-topics.sh --zookeeper {{zookeeper_host}}:2181 --create --topic {{env_name}}.{{item.name}} --partitions {{ item.num_of_partitions }} --replication-factor {{ item.replication_factor }} with_items: "{{ml_service_topics}}" ignore_errors: true when: kafka_id=="1" and ml_kafka_topic_create @@ -22,7 +22,7 @@ - processing-kafka - name: create topics - command: /opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic {{env_name}}.{{item.name}} --partitions {{ item.num_of_partitions }} --replication-factor {{ item.replication_factor }} + command: /opt/kafka/bin/kafka-topics.sh --zookeeper {{zookeeper_host}}:2181 --create --topic {{env_name}}.{{item.name}} --partitions {{ item.num_of_partitions }} --replication-factor {{ item.replication_factor }} with_items: "{{uci_service_topics}}" ignore_errors: true when: kafka_id=="1" and uci_kafka_topic_create @@ -30,7 +30,7 @@ - processing-kafka - name: create topics - command: /opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic {{env_name}}.{{item.name}} --partitions {{ item.num_of_partitions }} --replication-factor {{ item.replication_factor }} + command: /opt/kafka/bin/kafka-topics.sh --zookeeper {{zookeeper_host}}:2181 --create --topic {{env_name}}.{{item.name}} --partitions {{ item.num_of_partitions }} --replication-factor {{ item.replication_factor }} with_items: "{{av_service_topics}}" ignore_errors: true when: kafka_id=="1" and av_kafka_topic_create diff --git a/ansible/roles/stack-sunbird/defaults/main.yml b/ansible/roles/stack-sunbird/defaults/main.yml index 7fbeb17d70..581f28458b 100644 --- a/ansible/roles/stack-sunbird/defaults/main.yml +++ b/ansible/roles/stack-sunbird/defaults/main.yml @@ -73,6 +73,9 @@ sunbird_keycloak_client_id: 'portal' vault_sunbird_api_auth_token: sunbird_public_storage_account_name: sunbird_public_storage_account_key: +cloud_private_storage_accountname: +cloud_private_storage_secret: +cloud_private_storage_endpoint: sunbird_container_name: sunbird_portal_cdn_url: sunbird_dataservice_url: @@ -1070,3 +1073,4 @@ cloud_store_base_path_placeholder: "CLOUD_BASE_PATH" #Youtube Standard Licence Validation youtube_app_name: fetch-youtube-license youtube_api_key: "{{ lp_vault_youtube_api_key }}" +pgssl: "false" diff --git a/ansible/roles/stack-sunbird/templates/analytics_api_service.conf b/ansible/roles/stack-sunbird/templates/analytics_api_service.conf index 2afb0291de..8bdcd63467 100644 --- a/ansible/roles/stack-sunbird/templates/analytics_api_service.conf +++ b/ansible/roles/stack-sunbird/templates/analytics_api_service.conf @@ -7,7 +7,7 @@ data_exhaust.retry.limit="3" data_exhaust.dataset.list=["eks-consumption-raw", "eks-consumption-summary", "eks-consumption-metrics","eks-creation-raw", "eks-creation-summary", "eks-creation-metrics"] data_exhaust.dataset.default="eks-consumption-raw" data_exhaust.output_format="json" -data_exhaust.bucket="reports" +data_exhaust.bucket="{{ cloud_storage_privatereports_bucketname }}" cassandra.service.embedded.enable=false cassandra.keyspace_prefix="{{ cassandra.keyspace_prefix }}" device-register-controller-dispatcher { @@ -200,11 +200,12 @@ public { } } } -cloud_storage_type="azure" -storage.key.config="azure_storage_key" -storage.secret.config="azure_storage_secret" -public.storage.key.config="public_azure_storage_key" -public.storage.secret.config="public_azure_storage_secret" +cloud_storage_type="{{cloud_service_provider}}" +storage.key.config="{{cloud_public_storage_accountname}}" +storage.secret.config="{{cloud_public_storage_secret}}" +public.storage.key.config="{{cloud_public_storage_accountname}}" +public.storage.secret.config="{{cloud_public_storage_secret}}" +cloud_storage_endpoint_with_protocol="{{cloud_storage_url}}" metrics.time.interval.min=30 cache.refresh.time.interval.min=1440 redis.host="{{metadata_redis_host}}" diff --git a/ansible/roles/stack-sunbird/templates/content-service_application.conf b/ansible/roles/stack-sunbird/templates/content-service_application.conf index dc90131a1b..8e8fa56f35 100644 --- a/ansible/roles/stack-sunbird/templates/content-service_application.conf +++ b/ansible/roles/stack-sunbird/templates/content-service_application.conf @@ -485,6 +485,7 @@ cloud_storage_type: "{{ cloud_service_provider }}" cloud_storage_key: "{{ cloud_public_storage_accountname }}" cloud_storage_secret: "{{ cloud_public_storage_secret }}" cloud_storage_endpoint: "{{ cloud_public_storage_endpoint }}" +cloud_storage_proxy_host: "{{ cloud_public_storage_proxy }}" cloud_storage_container: "{{ cloud_storage_content_bucketname }}" cloud_storage_client_id: "{{ cloud_public_storage_client_id | default('') }}" cloud_storage_private_key_id: "{{ cloud_public_storage_private_key_id | default('') }}" diff --git a/ansible/roles/stack-sunbird/templates/odk.env b/ansible/roles/stack-sunbird/templates/odk.env index 64ca86b1ec..5fcd16ddd4 100644 --- a/ansible/roles/stack-sunbird/templates/odk.env +++ b/ansible/roles/stack-sunbird/templates/odk.env @@ -1,7 +1,11 @@ -DB_URL=jdbc:postgresql://{{uci_postgres_host}}:5432/{{uci_odk_postgres_database}}?user={{uci_postgres_user}}&password={{uci_postgres_password}} DB_USERNAME={{uci_postgres_user}} DB_PASSWORD={{uci_postgres_password}} DB_SCHEMA=public DB_NAME={{uci_odk_postgres_database}} DB_PORT=5432 -AGGREGATE_HOST= \ No newline at end of file +AGGREGATE_HOST= +{% if pgssl == "true" %} +DB_URL=jdbc:postgresql://{{uci_postgres_host}}:5432/{{uci_odk_postgres_database}}?user={{uci_postgres_user}}&password={{uci_postgres_password}}&sslmode=require +{% else %} +DB_URL=jdbc:postgresql://{{uci_postgres_host}}:5432/{{uci_odk_postgres_database}}?user={{uci_postgres_user}}&password={{uci_postgres_password}} +{% endif %} \ No newline at end of file diff --git a/ansible/roles/stack-sunbird/templates/sunbird_cert-service.env b/ansible/roles/stack-sunbird/templates/sunbird_cert-service.env index 19a9a6c46c..2008cb28dc 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_cert-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_cert-service.env @@ -11,3 +11,4 @@ ITEXT_LICENSE_PATH=/home/sunbird/itext_trail_license.xml PUBLIC_CLOUD_STORAGE_KEY={{cloud_public_storage_accountname}} PUBLIC_CLOUD_STORAGE_SECRET={{cloud_public_storage_secret}} PUBLIC_CONTAINER_NAME={{cloud_storage_certqr_bucketname}} +CLOUD_STORAGE_ENDPOINT={{cloud_public_storage_endpoint}} diff --git a/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env b/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env index 0318a4ed7e..93cd6331fc 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env @@ -31,6 +31,7 @@ sunbird_encryption_key={{sunbird_encryption_key}} sunbird_encryption_mode={{sunbird_encryption_mode}} sunbird_account_name={{cloud_public_storage_accountname}} sunbird_account_key={{cloud_public_storage_secret}} +sunbird_account_endpoint={{cloud_public_storage_endpoint}} sunbird_quartz_mode={{sunbird_sunbird_quartz_mode}} sunbird_web_url={{sunbird_web_url}} sunbird_msg_91_auth={{sunbird_msg_91_auth}} @@ -90,6 +91,7 @@ sunbird_gzip_enable={{sunbird_gzip_enable}} sunbird_gzip_size_threshold={{sunbird_gzip_size_threshold | default(262144)}} sunbird_analytics_blob_account_name={{cloud_private_storage_accountname}} sunbird_analytics_blob_account_key={{cloud_private_storage_secret}} +sunbird_analytics_blob_account_endpoint={{cloud_private_storage_account_endpoint}} # Optional for caching sunbird_cache_enable={{sunbird_cache_enable | default(false)}} # Set below variables if above true @@ -152,7 +154,7 @@ cloud_storage_dial_bucketname={{ cloud_storage_dial_bucketname | default('dial') cloud_storage_path_prefix_dial={{ cloudstorage_relative_path_prefix_dial | default('DIAL_STORAGE_BASE_PATH') }} cloud_store_base_path_placeholder={{ cloud_store_base_path_placeholder | default('CLOUD_BASE_PATH') }} - # Release-5.2.0 LR-325 sunbird_dial_service_base_url={{ dial_service_url | default('http://dial-service:9000') }} sunbird_dial_service_search_url=/dialcode/v3/search + diff --git a/ansible/roles/stack-sunbird/templates/sunbird_player.env b/ansible/roles/stack-sunbird/templates/sunbird_player.env index 6486898185..e0c21ee7a9 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_player.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_player.env @@ -208,6 +208,7 @@ cloud_private_storage_accountname={{cloud_private_storage_accountname | default( cloud_private_storage_secret={{cloud_private_storage_secret | default("")}} cloud_private_storage_region={{cloud_private_storage_region | default("ap-south-1")}} cloud_private_storage_project={{cloud_private_storage_project | default("")}} +cloud_private_storage_endpoint={{cloud_private_storage_endpoint | default("")}} cloud_storage_privatereports_bucketname={{cloud_storage_privatereports_bucketname | default("reports")}} cloud_storage_resourceBundle_bucketname={{cloud_storage_label_bucketname | default("label")}} cloud_storage_desktopCrash_bucketname={{cloud_storage_desktopcrash_bucketname | default("desktopappcrashlogs")}} diff --git a/ansible/uploadFAQs.yml b/ansible/uploadFAQs.yml index b38f2ff99a..9913741938 100644 --- a/ansible/uploadFAQs.yml +++ b/ansible/uploadFAQs.yml @@ -58,6 +58,21 @@ with_items: - "{{ source_folder.split(',') }}" when: cloud_service_provider == "gcloud" + + - name: upload batch of files to oci oss + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: upload-folder.yml + vars: + oss_bucket_name: "{{ cloud_storage_public_bucketname }}" + oss_path: "" + local_file_or_folder_path: "{{ playbook_dir }}/../utils/{{ item }}" + with_items: + - "{{ source_folder.split(',') }}" + when: cloud_service_provider == "oci" tags: - upload-faqs @@ -106,5 +121,20 @@ with_items: - "{{ source_folder.split(',') }}" when: cloud_service_provider == "gcloud" + + - name: upload batch of files to oci oss + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: upload-folder.yml + vars: + oss_bucket_name: "{{ cloud_storage_content_bucketname }}" + oss_path: "" + local_file_or_folder_path: "{{ playbook_dir }}/../utils/{{ item }}" + with_items: + - "{{ source_folder.split(',') }}" + when: cloud_service_provider == "oci" tags: - upload-RC-schema diff --git a/kubernetes/ansible/roles/deploy-player/tasks/main.yml b/kubernetes/ansible/roles/deploy-player/tasks/main.yml index 0aa27af993..28a757f224 100644 --- a/kubernetes/ansible/roles/deploy-player/tasks/main.yml +++ b/kubernetes/ansible/roles/deploy-player/tasks/main.yml @@ -77,6 +77,7 @@ shell: kubectl get deployments.apps {{ release_name }} -o json -n {{ namespace }} | jq -r '.spec.template.spec.containers[0].image | split("/")[-1]' register: deployed_image + - set_fact: metadata_image: "{{ image_name }}:{{ image_tag }}" diff --git a/kubernetes/ansible/roles/helm-deploy/tasks/main.yml b/kubernetes/ansible/roles/helm-deploy/tasks/main.yml index 16b62ee9d6..e960145c7a 100644 --- a/kubernetes/ansible/roles/helm-deploy/tasks/main.yml +++ b/kubernetes/ansible/roles/helm-deploy/tasks/main.yml @@ -135,6 +135,7 @@ shell: kubectl get daemonsets {{ release_name }} -o json -n {{ namespace }} | jq -r '.spec.template.spec.containers[].image | split("/")[-1]' register: image + - set_fact: deployed_image: "{{ image if image.stdout_lines | length > 0 else deployed_image }}" diff --git a/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml b/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml index 0dd8167be2..b59cebe9a0 100644 --- a/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml +++ b/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml @@ -39,8 +39,9 @@ data: server { listen 80; listen [::]:80; - server_name {{ .Values.nginx_private_ingress_ip }}; - +{{- if and .Values.nginx_private_ingress_ip (ne .Values.csp "oci") }} + server_name: {{ .Values.nginx_private_ingress_ip }}; +{{- end }} resolver {{ .Values.kube_dns_ip }}; location /learner/ { @@ -125,7 +126,9 @@ data: set $target http://report-service.{{ .Values.namespace }}.svc.cluster.local:3030; rewrite ^/report/(.*) /$1 break; proxy_http_version 1.1; +{{- if and .Values.nginx_private_ingress_ip (ne .Values.csp "oci") }} proxy_set_header Host $server_name; +{{- end }} proxy_pass $target; } location /search/ { @@ -244,7 +247,9 @@ data: set $target http://registry-service.{{ .Values.namespace }}.svc.cluster.local:8081; rewrite ^/registry-service/(.*) /$1 break; proxy_http_version 1.1; +{{- if and .Values.nginx_private_ingress_ip (ne .Values.csp "oci") }} proxy_set_header Host $server_name; +{{- end }} proxy_pass $target; } location /ml-projects/ { diff --git a/kubernetes/helm_charts/core/nginx-private-ingress/values.j2 b/kubernetes/helm_charts/core/nginx-private-ingress/values.j2 index c1c2491446..393476d491 100644 --- a/kubernetes/helm_charts/core/nginx-private-ingress/values.j2 +++ b/kubernetes/helm_charts/core/nginx-private-ingress/values.j2 @@ -38,3 +38,4 @@ autoscaling: targetCPUUtilizationPercentage: {{ nginx_private_ingress_autoscaling_targetCPUUtilizationPercentage|default(60) }} targetMemoryUtilizationPercentage: {{ nginx_private_ingress_autoscaling_targetMemoryUtilizationPercentage|default('') }} +csp: {{cloud_service_provider}} \ No newline at end of file diff --git a/kubernetes/helm_charts/logging/fluent-bit/templates/serviceaccount.yaml b/kubernetes/helm_charts/logging/fluent-bit/templates/serviceaccount.yaml index f162d2bc28..83329448f6 100644 --- a/kubernetes/helm_charts/logging/fluent-bit/templates/serviceaccount.yaml +++ b/kubernetes/helm_charts/logging/fluent-bit/templates/serviceaccount.yaml @@ -4,7 +4,11 @@ metadata: name: fluent-bit namespace: {{ default .Values.namespace .Release.Namespace }} --- +{{- if (.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1") }} +apiVersion: rbac.authorization.k8s.io/v1 +{{ else }} apiVersion: rbac.authorization.k8s.io/v1beta1 +{{- end }} kind: ClusterRole metadata: name: fluent-bit-read @@ -15,7 +19,11 @@ rules: - pods verbs: ["get", "list", "watch"] --- +{{- if (.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1") }} +apiVersion: rbac.authorization.k8s.io/v1 +{{ else }} apiVersion: rbac.authorization.k8s.io/v1beta1 +{{- end }} kind: ClusterRoleBinding metadata: name: fluent-bit-read diff --git a/pipelines/deploy/ContentFramework/Jenkinsfile b/pipelines/deploy/ContentFramework/Jenkinsfile index a02c72eb69..f85dc9b7d3 100644 --- a/pipelines/deploy/ContentFramework/Jenkinsfile +++ b/pipelines/deploy/ContentFramework/Jenkinsfile @@ -44,6 +44,7 @@ node() { sh """ zip -r content-editor-artifact.zip ansible/content-editor cd ansible/content-editor + nvm use 6 sudo npm install -g gulp npm install npm install promise diff --git a/private_repo/ansible/inventory/dev/Core/common.yml b/private_repo/ansible/inventory/dev/Core/common.yml index aef622d8cc..3386020d08 100644 --- a/private_repo/ansible/inventory/dev/Core/common.yml +++ b/private_repo/ansible/inventory/dev/Core/common.yml @@ -204,7 +204,12 @@ monitor_alerts_mail_from_email: "{{ sunbird_mail_server_from_email }}" ekstep_s3_env: "" # This variable is not used and leave the value as empty freshDesk_token: "" -# Below endpoint is not required in current release + +# provide the s3 compatible endpoint +# for AWS +# cloud_public_storage_endpoint: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" +# for OCI +#cloud_public_storage_endpoint: "https://.compat.objectstorage.{{cloud_public_storage_region}}.oraclecloud.com" cloud_public_storage_endpoint: "" # Update below vars if seperate object storage is required @@ -238,7 +243,7 @@ cloud_artifact_storage_project: "{{ cloud_public_storage_project }}" # Building block vars cloud_storage_base_url: "{{ cloud_storage_url }}" cloudstorage_base_path: "{{ cloud_storage_url }}" -valid_cloudstorage_base_urls: '["{{ cloud_storage_url }}"]' +valid_cloudstorage_base_urls: '["{{cloud_storage_url}}"]' cloudstorage_relative_path_prefix: "CONTENT_STORAGE_BASE_PATH" # Provide the admin-api consumer access to all API's - The public repo restricts this for security reasons diff --git a/private_repo/ansible/inventory/dev/DataPipeline/common.yml b/private_repo/ansible/inventory/dev/DataPipeline/common.yml index 971c04935c..c5790ec42c 100644 --- a/private_repo/ansible/inventory/dev/DataPipeline/common.yml +++ b/private_repo/ansible/inventory/dev/DataPipeline/common.yml @@ -136,7 +136,12 @@ imagepullsecrets: "{{env}}registrysecret" # kubernetes imagePul kubeconfig_path: /var/lib/jenkins/secrets/k8s.yaml # kubeconfig file path on jenkins core_kubeconfig_path: "{{ kubeconfig_path }}" # kubeconfig file path on jenkins for core kube cluster, change this if you use separate kube cluster for core and KP + DP -# Below endpoint is not required in current release +# provide the s3 compatible endpoint +# for AWS +# cloud_public_storage_endpoint: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" +# for OCI +#cloud_public_storage_endpoint: "https://.compat.objectstorage.{{cloud_public_storage_region}}.oraclecloud.com" + cloud_public_storage_endpoint: "" # Update below vars if seperate object storage is required diff --git a/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml b/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml index 86b9e1eb9a..ce4e93b37e 100644 --- a/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml +++ b/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml @@ -124,7 +124,12 @@ kp_schema_base_path: "{{ cloud_storage_url }}/{{ cloud_storage_content_bucketnam imagepullsecrets: "{{env}}registrysecret" # kubernetes imagePullSecrets kubeconfig_path: /var/lib/jenkins/secrets/k8s.yaml # kubeconfig file path on jenkins -# Below endpoint is not required in current release +# provide the s3 compatible endpoint +# for AWS +# cloud_public_storage_endpoint: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" +# for OCI +#cloud_public_storage_endpoint: "https://.compat.objectstorage.{{cloud_public_storage_region}}.oraclecloud.com" + cloud_public_storage_endpoint: "" # Update below vars if seperate object storage is required @@ -163,7 +168,7 @@ cloud_storage_base_url: "{{ cloud_storage_url }}" cloudstorage_base_path: "{{ cloud_storage_url }}" valid_cloudstorage_base_urls: '["{{ cloud_storage_url }}"]' cloudstorage_relative_path_prefix: "CONTENT_STORAGE_BASE_PATH" -cloud_storage_pathstyle_access: false +cloud_storage_pathstyle_access: false ### Lern BB - Adding Lern specific vars here. In future if we want to move it to seperate folder this can be used as the starting point From d2b113d6975ba44e64e0ec6843a7f6649d5072f4 Mon Sep 17 00:00:00 2001 From: Abhishek P N <116337484+abhishekpnt@users.noreply.github.com> Date: Tue, 22 Aug 2023 17:18:08 +0530 Subject: [PATCH 44/53] Issue #ED-2632 fix: removed duplicate variables (#3887) * Issue #ED-2632 fix: removed unused env variables and added OCI endpoint variable * Issue #ED-2632 cleanup: Removed Azure related variables * Issue #ED-2632 cleanup: changed added storage account url * Issue #ED-2632 fix: removed duplicate variables --- ansible/roles/stack-sunbird/templates/sunbird_player.env | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ansible/roles/stack-sunbird/templates/sunbird_player.env b/ansible/roles/stack-sunbird/templates/sunbird_player.env index e0c21ee7a9..f9cbbdc54d 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_player.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_player.env @@ -214,5 +214,4 @@ cloud_storage_resourceBundle_bucketname={{cloud_storage_label_bucketname | defau cloud_storage_desktopCrash_bucketname={{cloud_storage_desktopcrash_bucketname | default("desktopappcrashlogs")}} #release-6.0.0 -sunbird_default_board={{sunbird_default_board | default("")}} -cloud_private_storage_endpoint={{cloud_private_storage_endpoint | default("")}} \ No newline at end of file +sunbird_default_board={{sunbird_default_board | default("")}} \ No newline at end of file From 922167273f5522cc24723966ddd15cb7c5704c7b Mon Sep 17 00:00:00 2001 From: Jayaprakash8887 Date: Wed, 23 Aug 2023 13:03:02 +0530 Subject: [PATCH 45/53] Issue #LR-539 feat: configuration reconcile (#3888) Co-authored-by: Jayaprakash8887 --- .../roles/stack-sunbird/templates/sunbird_lms-service.env | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env b/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env index 93cd6331fc..5be4a2f3c8 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env @@ -158,3 +158,10 @@ cloud_store_base_path_placeholder={{ cloud_store_base_path_placeholder | default sunbird_dial_service_base_url={{ dial_service_url | default('http://dial-service:9000') }} sunbird_dial_service_search_url=/dialcode/v3/search +# Release-5.3.0 LR-539 +exhaust_api_base_url={{ analytics_service_url | default('http://analytics-service:9000') }} +exhaust_api_submit_endpoint=/request/submit +exhaust_api_list_endpoint=/request/list/ + +# Release-5.3.0 LR-324 +content_read_url=/content/v3/read/ \ No newline at end of file From 408c8c4898e5a16b569342ff190d5fa56587ae6c Mon Sep 17 00:00:00 2001 From: santhosh-tg <93243580+santhosh-tg@users.noreply.github.com> Date: Thu, 14 Sep 2023 09:02:23 +0530 Subject: [PATCH 46/53] Fix syntax (#3894) --- .../core/nginx-private-ingress/templates/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml b/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml index b59cebe9a0..4cb69fd90f 100644 --- a/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml +++ b/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml @@ -40,7 +40,7 @@ data: listen 80; listen [::]:80; {{- if and .Values.nginx_private_ingress_ip (ne .Values.csp "oci") }} - server_name: {{ .Values.nginx_private_ingress_ip }}; + server_name {{ .Values.nginx_private_ingress_ip }}; {{- end }} resolver {{ .Values.kube_dns_ip }}; From d6c687564e4d5a81cc567d753ce49ccd3d09bbd1 Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Tue, 19 Sep 2023 11:21:54 +0530 Subject: [PATCH 47/53] Updated vars for cron schedule (#3895) * Updated vars for cron schedule * Update main.yml --- ansible/roles/ml-analytics-service/defaults/main.yml | 5 +++++ ansible/roles/ml-analytics-service/tasks/main.yml | 11 ++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index 15ef432e43..a67906b1f8 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -133,3 +133,8 @@ ml_analytics_reports_store: "{{ cloud_service_provider }}" ml_analytics_reports_container: "{{ cloud_storage_privatereports_bucketname }}" ml_analytics_driver_memory: "{{ ml_analytics_default_driver_memory | default('5g') }}" ml_analytics_executor_memory: "{{ ml_analytics_default_executor_memory | default('5g') }}" +ml_analytics_batch_cron_minute: "{{ ml_batch_cron_minute | default('30') }}" +ml_analytics_batch_cron_hour: "{{ ml_batch_cron_hour | default('18') }}" +ml_analytics_nvsk_cron_minute: "{{ ml_nvsk_cron_minute | default('30') }}" +ml_analytics_nvsk_cron_hour: "{{ ml_nvsk_cron_hour | default('7') }}" +ml_analytics_nvsk_cron_weekday: "{{ ml_nvsk_cron_weekday | default('4') }}" diff --git a/ansible/roles/ml-analytics-service/tasks/main.yml b/ansible/roles/ml-analytics-service/tasks/main.yml index 6f37d6b3ea..7fbd3fe1ba 100755 --- a/ansible/roles/ml-analytics-service/tasks/main.yml +++ b/ansible/roles/ml-analytics-service/tasks/main.yml @@ -26,6 +26,7 @@ - python3-virtualenv - zip - unzip + - acl state: present - name: CHANGE THE OWNERSHIP FOR THIS {{ BASEPATH }} DIRECTORY @@ -135,15 +136,15 @@ cron: name: "Run Batch Ingestion Job" user: "{{ USER }}" - minute: "30" - hour: "18" + minute: "{{ ml_analytics_batch_cron_minute }}" + hour: "{{ ml_analytics_batch_cron_hour }}" job: "{{ BASEPATH }}/ml-analytics-service/run.sh > {{ BASEPATH }}/ml-analytics-service/crontab_job.log" - name: CREATE THE CRON NVSK Data Upload cron: name: "NVSK Data Upload JOB" user: "{{ USER }}" - minute: "30" - hour: "7" - weekday: "4" + minute: "{{ ml_analytics_nvsk_cron_minute }}" + hour: "{{ ml_analytics_nvsk_cron_hour }}" + weekday: "{{ ml_analytics_nvsk_cron_weekday }}" job: "{{ BASEPATH }}/ml-analytics-service/run_weekly.sh > {{ BASEPATH }}/ml-analytics-service/nvsk_data_weekly.logs" From 8dc81f6fd96045eb7a727bd104c28395534cc5c5 Mon Sep 17 00:00:00 2001 From: santhosh-tg <93243580+santhosh-tg@users.noreply.github.com> Date: Mon, 25 Sep 2023 14:15:56 +0530 Subject: [PATCH 48/53] [ED-2793] Fix: release-6.0.0 installation issues (#3900) * Update Jenkins version * Fix job parameter * Fix error: You need to install jmespath prior to running json_query filter --- deploy/jenkins/jenkins-server-setup.sh | 5 +++-- .../dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/deploy/jenkins/jenkins-server-setup.sh b/deploy/jenkins/jenkins-server-setup.sh index d96e3b4228..739fb138dc 100755 --- a/deploy/jenkins/jenkins-server-setup.sh +++ b/deploy/jenkins/jenkins-server-setup.sh @@ -12,10 +12,10 @@ echo -e "\n\e[0;32m${bold}Installating JDK8${normal}\n" apt-get install -y openjdk-8-jdk echo -e "\n\e[0;32m${bold}Installating Jenkins${normal}" -wget -q -O - https://pkg.jenkins.io/debian-stable/jenkins.io.key | apt-key add - +wget -q -O - https://pkg.jenkins.io/debian-stable/jenkins.io-2023.key | apt-key add - apt-add-repository "deb https://pkg.jenkins.io/debian-stable binary/" apt-get update -apt-get install -y jenkins=2.319.3 +apt-get install -y jenkins=2.346.3 echo -e "\n\e[0;32m${bold}Installating PIP${normal}" apt-get install -y python-pip @@ -115,6 +115,7 @@ su jenkins bash -c "curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34. echo -e "\n\e[0;32m${bold}Installing jmespath${normal}" sudo apt install -y python3-jmespath +sudo apt install python-jmespath #python2 #### Kubernetes Tools #### diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml index 859b804b88..f669caa862 100644 --- a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml @@ -26,7 +26,7 @@ @@ -114,4 +114,4 @@ return """<b>This parameter is not used</b>""" false - \ No newline at end of file + From 6320664a744ec39c40cb543af5c26b24fc15c6e6 Mon Sep 17 00:00:00 2001 From: santhosh-tg <93243580+santhosh-tg@users.noreply.github.com> Date: Mon, 25 Sep 2023 14:58:21 +0530 Subject: [PATCH 49/53] Update main.yml with cassandra (#3845) (#3901) Co-authored-by: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> --- ansible/roles/cassandra/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/cassandra/defaults/main.yml b/ansible/roles/cassandra/defaults/main.yml index 33c02c17bf..8f1f0f7951 100644 --- a/ansible/roles/cassandra/defaults/main.yml +++ b/ansible/roles/cassandra/defaults/main.yml @@ -1,5 +1,5 @@ user_home: "/home/{{ ansible_ssh_user }}/" -cassandra_repo: 'deb http://www.apache.org/dist/cassandra/debian 311x main' -cassandra_repo_key: 'https://www.apache.org/dist/cassandra/KEYS' +cassandra_repo: 'deb https://debian.cassandra.apache.org 311x main' +cassandra_repo_key: 'https://downloads.apache.org/cassandra/KEYS' resource_crunch: "yes" # make this value yes if heap size should be quarter of the Server Memory when you are deploying multiple databases, else make this value no to have 50% of the value as per the best practice of cassandra cassandra_home_directory: "/var/lib/cassandra" From 6ce00cfa13d2419dcb93aff02f66a65a4adfffd0 Mon Sep 17 00:00:00 2001 From: Prasath Sivasubramaniyan Date: Tue, 26 Sep 2023 16:10:34 +0530 Subject: [PATCH 50/53] ED-2838: ##Release-6.0.0 new variables (#3899) * ED-2838 Release-6.0.0 new variables * Added with comments * ED-2838 jenkins josb update and new variables * added storage endpoint detials --- .../Build/jobs/Core/jobs/Analytics/config.xml | 31 +++++++++++++++---- .../Build/jobs/Core/jobs/Content/config.xml | 18 +++++++++++ .../Build/jobs/Core/jobs/Learner/config.xml | 20 +++++++++++- .../jobs/Build/jobs/Core/jobs/Lms/config.xml | 18 +++++++++++ .../jobs/AnalyticsCore/config.xml | 26 +++++++++++++++- .../jobs/CoreDataProducts/config.xml | 23 +++++++++++++- .../jobs/FlinkJobs/config.xml | 18 +++++++++++ .../Lern/jobs/LernDataProducts/config.xml | 25 +++++++++++---- .../jobs/Lern/jobs/LernFlinkJobs/config.xml | 18 +++++++++++ .../ansible/inventory/dev/Core/common.yml | 9 ++++++ .../ansible/inventory/dev/Core/secrets.yml | 2 +- .../dev/KnowledgePlatform/common.yml | 8 ++++- 12 files changed, 199 insertions(+), 17 deletions(-) diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml index 5bc1fe4d34..c6304ef3b4 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml @@ -35,6 +35,30 @@ refs/heads/${public_repo_branch} true + + CLOUD_STORE_GROUP_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk group id</li> +</b></font> + org.sunbird + false + + + CLOUD_STORE_ARTIFACT_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk artifact id</li> +</b></font> + cloud-store-sdk_2.12 + false + + + CLOUD_STORE_VERSION + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk version</li> +</b></font> + 1.4.0 + false + @@ -48,12 +72,7 @@ - - - H/15 * * * * - false - - + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml index 7b47e1972e..f3f5d6477c 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + CLOUD_STORE_GROUP_ID + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + CLOUD_STORE_ARTIFACT_ID + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12 + ${cloud_store_artifact_id} + false + + + CLOUD_STORE_VERSION + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml index 2cdac260ad..a12dd235f1 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + cloud_store_group_id + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + cloud_store_artifact_id + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk + ${cloud_store_artifact_id} + false + + + cloud_store_version + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + @@ -51,7 +69,7 @@ - H/15 * * * * + false diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml index 89116c8406..4eecb95a61 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + cloud_store_group_id + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + cloud_store_artifact_id + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk + ${cloud_store_artifact_id} + false + + + cloud_store_version + Set the Cloud store sdk version. e.g 1.4.6 + ${cloud_store_version} + false + diff --git a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/AnalyticsCore/config.xml b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/AnalyticsCore/config.xml index 2b9e72c45c..5c202a759c 100644 --- a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/AnalyticsCore/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/AnalyticsCore/config.xml @@ -36,6 +36,30 @@ refs/heads/${public_repo_branch} true + + CLOUD_STORE_GROUP_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk group id</li> +</b></font> + org.sunbird + false + + + CLOUD_STORE_ARTIFACT_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk artifact id</li> +</b></font> + cloud-store-sdk_2.12 + false + + + CLOUD_STORE_VERSION + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk version</li> +</b></font> + 1.4.0 + false + @@ -51,7 +75,7 @@ - H/15 * * * * + false diff --git a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/CoreDataProducts/config.xml b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/CoreDataProducts/config.xml index a458418b24..8ff4dc5426 100644 --- a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/CoreDataProducts/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/CoreDataProducts/config.xml @@ -27,6 +27,27 @@ github_release_tag + <font color=red size=2><b>CAUTION: If the value is blank, latest code will be built. Specify github tag name to build from a tag.</b></font> + false + + + CLOUD_STORE_GROUP_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk group id</li> +</b></font> + org.sunbird + false + + + CLOUD_STORE_ARTIFACT_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk artifact id</li> +</b></font> + cloud-store-sdk_2.12 + false + + + CLOUD_STORE_VERSION <font style="color:dimgray;font-size:14px;"><b> <li>To build from a tag, use refs/tags/github_tag</li> <li>To build from a branch, use refs/heads/github_branch</li> @@ -51,7 +72,7 @@ - H/15 * * * * + false diff --git a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml index 11f30a0d33..62a0cd1f68 100644 --- a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + CLOUD_STORE_GROUP_ID + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + CLOUD_STORE_ARTIFACT_ID + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12 + ${cloud_store_artifact_id} + false + + + CLOUD_STORE_VERSION + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + diff --git a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml index 088b9fa27f..9b997d2ce1 100644 --- a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + cloud_store_group_id + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + cloud_store_artifact_id + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12 + ${cloud_store_artifact_id} + false + + + cloud_store_version + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + @@ -49,12 +67,7 @@ - - - H/15 * * * * - false - - + diff --git a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml index df82bece0b..f8041941e2 100644 --- a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + cloud_store_group_id + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + cloud_store_artifact_id + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12 + ${cloud_store_artifact_id} + false + + + cloud_store_version + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + diff --git a/private_repo/ansible/inventory/dev/Core/common.yml b/private_repo/ansible/inventory/dev/Core/common.yml index 3386020d08..da2d8a9694 100644 --- a/private_repo/ansible/inventory/dev/Core/common.yml +++ b/private_repo/ansible/inventory/dev/Core/common.yml @@ -488,3 +488,12 @@ ml_analytics_cname_url: # Reference value - https://obj.dev.sunbirded.org/samiks ml_analytics_client_id : project-sunbird-dev-client # Keycloak client group ml_analytics_username : reportAdmin1 # Username of a user which has roles of "PROGRAM_MANAGER", "PROGRAM_DESIGNER", "REPORT_ADMIN", "REPORT_VIEWER" ml_analytics_createdBy : fb85a044-d9eb-479b-a55a-faf1bfaea14d # Unique system generated user UUID which is the same user as above +ml_container: samiksha + +# provide the s3 compatible endpoint +# for AWS +# cloud_private_storage_account_endpoint: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" +# for OCI +#cloud_private_storage_account_endpoint: "https://.compat.objectstorage.{{cloud_public_storage_region}}.oraclecloud.com" +cloud_private_storage_account_endpoint: "{{ cloud_public_storage_endpoint }}" # Leave Blank for Azure +bb_name: "" #Leave Blank diff --git a/private_repo/ansible/inventory/dev/Core/secrets.yml b/private_repo/ansible/inventory/dev/Core/secrets.yml index 2e16859454..c7f3e5b7e4 100644 --- a/private_repo/ansible/inventory/dev/Core/secrets.yml +++ b/private_repo/ansible/inventory/dev/Core/secrets.yml @@ -237,4 +237,4 @@ lp_vault_youtube_api_key: # youtube api token if you want # Examples values added below ml_analytics_authorization_access_token : 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJkOTY3NzRjYzXXXXXXXXXXXXXQ4Y2RiOWQ2Mzg0OSJ9.n4hXxKxl_698yeZPSWhXeGvMOb4esfgPadIZe8jZ0Z4' # Bearer auth token which has access to APIs as defined in the release note ml_analytics_password : Test@123 # Password of a user which has roles of "PROGRAM_MANAGER", "PROGRAM_DESIGNER", "REPORT_ADMIN", "REPORT_VIEWER" matching with supplied username to variable ml_analytics_username -ml_analytics_client_secret : fd241dce-4ZZ9-47e1-97cf-1c7de7a44216 # Keycloak client secret for creating tokens, mapped to client id supplied to variable ml_analytics_client_id +ml_analytics_client_secret : fd241dce-4ZZ9-47e1-97cf-1c7de7a44216 # Keycloak client secret for creating tokens, mapped to client id supplied to variable ml_analytics_client_id \ No newline at end of file diff --git a/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml b/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml index ce4e93b37e..d87400b63c 100644 --- a/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml +++ b/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml @@ -213,4 +213,10 @@ postgres: db_admin_password: "{{dp_vault_pgdb_admin_password}}" # graylog -send_logs_to_graylog: true # filebeat agents will send logs to graylog instead of ES \ No newline at end of file +send_logs_to_graylog: true # filebeat agents will send logs to graylog instead of ES + +druid_storage_type: "" # Your cloud service provider name. Supported values are aws, azure, gcloud + +# Optional variables - Can be left blank +cloud_storage_report_verfication_bucketname: "" +dp_storage_endpoint_config: "" From c1b8c59638ce85b5ac81ca1a3e79293a61d41ea9 Mon Sep 17 00:00:00 2001 From: santhosh-tg Date: Thu, 28 Sep 2023 11:43:00 +0530 Subject: [PATCH 51/53] Update maven repo url --- deploy/jenkins/jenkins-server-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/jenkins/jenkins-server-setup.sh b/deploy/jenkins/jenkins-server-setup.sh index 739fb138dc..2c7516a57a 100755 --- a/deploy/jenkins/jenkins-server-setup.sh +++ b/deploy/jenkins/jenkins-server-setup.sh @@ -156,7 +156,7 @@ rm openjdk-11.0.2_linux-x64_bin.tar.gz #Install maven 3.6.3 echo -e "\n\e[0;32m${bold}Installating maven 3.6.3${normal}" -wget https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz +wget https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz tar -xf apache-maven-3.6.3-bin.tar.gz mv apache-maven-3.6.3 /opt/ mv /opt/apache-maven-3.6.3/bin/mvn /opt/apache-maven-3.6.3/bin/mvn3.6 From 7ae3271bd2d928218abdd8f41a32c6945867e07b Mon Sep 17 00:00:00 2001 From: santhosh-tg Date: Thu, 28 Sep 2023 20:27:16 +0530 Subject: [PATCH 52/53] Add missing jobs --- .../jobs/InquiryFlinkJob/config.xml | 108 ++++++++++++ .../jobs/Core/jobs/Assessment/config.xml | 55 ++++-- .../jobs/InquiryFlinkJob/config.xml | 77 +++++++++ .../jobs/InquiryFlinkJob/config.xml | 149 ++++++++++++++++ .../Kubernetes/InquiryUploadSchema/config.xml | 163 ++++++++++++++++++ 5 files changed, 534 insertions(+), 18 deletions(-) create mode 100644 deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml create mode 100644 deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml create mode 100644 deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml create mode 100644 deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/InquiryUploadSchema/config.xml diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml new file mode 100644 index 0000000000..28b220559d --- /dev/null +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml @@ -0,0 +1,108 @@ + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + 10 + -1 + 1 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + Build/KnowledgePlatform/InquiryFlinkJob + false + + + image_tag + <font color=darkgreen size=2><b>OPTIONAL: Specify the tag to upload a specific image version to the container registry.</b></font> + + false + + + artifact_source + <font color=dimgray size=2><b> +ArtifactRepo - Push the docker image to container registry. +</b></font> + + + ArtifactRepo + + + + + + + 0 + 0 + + false + project + false + + + + + + + + Build/KnowledgePlatform/InquiryFlinkJob + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/Sunbird-inQuiry/data-pipeline.git + + + + + ${inquiry_pipeline_branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + kubernetes/pipelines/upload/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml index d2c038b5fa..b4deb5e46b 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml @@ -1,6 +1,6 @@ - + hudson.model.ParametersDefinitionProperty com.sonyericsson.rebuild.RebuildSettings @@ -19,26 +19,47 @@ - + false false - github_release_tag + inquiry_release_tag <font style="color:dimgray;font-size:14px;"><b> <li>To build from a tag, use refs/tags/github_tag</li> <li>To build from a branch, use refs/heads/github_branch</li> -<li>The default value of ${public_repo_branch} will be the release / tag version set in global configuration</li> -<li>To build from a differnt branch, replace the ${public_repo_branch} with your branch</li> +<li>The default value of ${inquiry_service_build_branch_or_tag} will be the release / tag version set in global configuration</li> +<li>To build from a differnt branch, replace the ${inquiry_service_build_branch_or_tag} with your branch</li> </b></font> - refs/heads/${public_repo_branch} + refs/heads/${inquiry_service_build_branch_or_tag} true + + core_release_tag + <font style="color:dimgray;font-size:14px;"><b> +<li>Provide this value to build core components from Knowlg BB</li> +<li>To build from a tag, use refs/tags/github_tag</li> +<li>To build from a branch, use refs/heads/github_branch</li> +<li>The default value of ${inquiry_core_build_branch_or_tag} will be the release / tag version set in global configuration</li> +<li>To build from a differnt branch, replace the ${inquiry_core_build_branch_or_tag} with your branch</li> +</b></font> + refs/heads/${inquiry_core_build_branch_or_tag} + false + + + core_repo_link + Knowlg Core Repo + + + https://github.com/project-sunbird/knowledge-platform.git + + + - + 0 0 @@ -49,31 +70,29 @@ - - - H/15 * * * * - false - - + - - + + 2 - https://github.com/project-sunbird/knowledge-platform.git + https://github.com/Sunbird-inQuiry/inquiry-api-service.git - ${github_release_tag} + ${inquiry_release_tag} + false + + build/assessment-service/Jenkinsfile false false - \ No newline at end of file + diff --git a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml new file mode 100644 index 0000000000..93b0eba8f7 --- /dev/null +++ b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml @@ -0,0 +1,77 @@ + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + 10 + -1 + 1 + + + + + false + false + + + + + inquiry_release_tag + <font style="color:dimgray;font-size:14px;"><b> +<li>To build from a tag, use refs/tags/github_tag</li> +<li>To build from a branch, use refs/heads/github_branch</li> +<li>The default value of ${inquiry_pipeline_branch_or_tag} will be the release / tag version set in global configuration</li> +<li>To build from a differnt branch, replace the ${inquiry_pipeline_branch_or_tag} with your branch</li> +</b></font> + refs/heads/${inquiry_pipeline_branch_or_tag} + true + + + + + 0 + 0 + + false + project + false + + + + + + + + + + 2 + + + https://github.com/Sunbird-inQuiry/data-pipeline.git + + + + + ${inquiry_release_tag} + + + false + + + + kubernetes/pipelines/build/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml new file mode 100644 index 0000000000..af888dbe75 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml @@ -0,0 +1,149 @@ + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + 10 + -1 + 2 + + + + + false + false + + + + + private_branch + + choice-parameter-2544395024638227 + 1 + + true + + + + true + + + InquiryFlinkJob + Deploy/dev/KnowledgePlatform/InquiryFlinkJob + + + ET_FORMATTED_HTML + true + + + inquiry_release_tag + <font style="color:dimgray;font-size:14px;"><b> +<li>To build from a tag, use refs/tags/github_tag</li> +<li>To build from a branch, use refs/heads/github_branch</li> +<li>The default value of ${inquiry_pipeline_branch_or_tag} will be the release / tag version set in global configuration</li> +<li>To build from a differnt branch, replace the ${inquiry_pipeline_branch_or_tag} with your branch</li> +</b></font> + refs/heads/${inquiry_pipeline_branch_or_tag} + false + + + image_tag + <font color=red size=2><b>CAUTION: If the value is blank, image tag will be taken from the latest metadata.json.</b></font> + + false + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + ArtifactUpload/dev/KnowledgePlatform/InquiryFlinkJob + false + + + job_names_to_deploy + <font color=green size=2><b>Choose the job names to deploy. Multi-selection is available.</b></font> + choice-parameter-1273072434092073 + 1 + + true + + + + InquiryFlinkJob + Deploy/dev/KnowledgePlatform/InquiryFlinkJob + + + PT_MULTI_SELECT + false + 1 + + + + + 0 + 0 + + false + project + false + + + + + + + + + + 2 + + + https://github.com/Sunbird-inQuiry/data-pipeline.git + + + + + ${inquiry_release_tag} + + + false + + + + true + false + + 0 + false + + + + kubernetes/pipelines/deploy/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/InquiryUploadSchema/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/InquiryUploadSchema/config.xml new file mode 100644 index 0000000000..7441f81065 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/InquiryUploadSchema/config.xml @@ -0,0 +1,163 @@ + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + private_branch + + choice-parameter-2544395024638227 + 1 + + true + + + + true + + + InquiryUploadSchema + Deploy/dev/Kubernetes/InquiryUploadSchema + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-2620434998790477 + 1 + + true + + + + true + + + InquiryUploadSchema + Deploy/dev/Kubernetes/InquiryUploadSchema + + + ET_FORMATTED_HTML + true + + + schema_repo + <font color=dimgray size=2><b>Enter the repo url from which schema folder to be uploaded</b></font> + + + https://github.com/Sunbird-inQuiry/inquiry-api-service.git + https://github.com/project-sunbird/knowledge-platform.git + + + + + schema_repo_branch_or_tag + <font color=dimgray size=2><b>Enter the branch or tag for schema repo</b></font> + + false + + + source_folder + + choice-parameter-5348290139923778 + 1 + + true + + + + InquiryUploadSchema + Deploy/dev/Kubernetes/InquiryUploadSchema + + + PT_MULTI_SELECT + false + 1 + + + + + 0 + 0 + + false + project + false + + + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/upload/schema/knowledge-platform/schema.Jenkinsfile + false + + + false + From 22f4afa760a2c8ffcb995df6879d473b67129f6a Mon Sep 17 00:00:00 2001 From: santhosh-tg Date: Fri, 29 Sep 2023 14:07:41 +0530 Subject: [PATCH 53/53] Fix docker creds secrets issue when having specialcharacters --- kubernetes/ansible/bootstrap.yaml | 2 +- kubernetes/ansible/bootstrap_minimal.yaml | 2 +- kubernetes/ansible/namespace.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/ansible/bootstrap.yaml b/kubernetes/ansible/bootstrap.yaml index 0fa7e0acfe..72f1beacff 100644 --- a/kubernetes/ansible/bootstrap.yaml +++ b/kubernetes/ansible/bootstrap.yaml @@ -39,7 +39,7 @@ - name: "Tagging {{ namespace }} to enable istio" shell: "kubectl label namespaces {{ namespace }} istio-injection=enabled --overwrite" - name: Creating docker secrets - shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ namespace }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password {{ vault_docker_registry_password }} --dry-run=client -o=yaml | kubectl apply -f -" + shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ namespace }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password '{{ vault_docker_registry_password }}' --dry-run=client -o=yaml | kubectl apply -f -" - name: Installing reloader for configmaps reload shell: helm upgrade --install --atomic reloader ../helm_charts/core/reloader --namespace "{{ namespace }}" -f /tmp/helm_vars.yaml roles: diff --git a/kubernetes/ansible/bootstrap_minimal.yaml b/kubernetes/ansible/bootstrap_minimal.yaml index 44170246ae..cdb145423e 100644 --- a/kubernetes/ansible/bootstrap_minimal.yaml +++ b/kubernetes/ansible/bootstrap_minimal.yaml @@ -30,7 +30,7 @@ - logging ignore_errors: yes - name: Creating docker secrets - shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password {{ vault_docker_registry_password }} --dry-run=client -o=yaml | kubectl apply -f -" + shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password '{{ vault_docker_registry_password }}' --dry-run=client -o=yaml | kubectl apply -f -" when: imagepullsecrets|length > 0 with_items: - "{{ bootstrap_namespace.split(',') }}" diff --git a/kubernetes/ansible/namespace.yml b/kubernetes/ansible/namespace.yml index bc62234c2c..bc38fab7d8 100644 --- a/kubernetes/ansible/namespace.yml +++ b/kubernetes/ansible/namespace.yml @@ -28,7 +28,7 @@ - "{{ bootstrap_namespace.split(',') }}" ignore_errors: yes - name: Creating docker secrets - shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password {{ vault_docker_registry_password }} --dry-run=client -o=yaml | kubectl apply -f -" + shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password '{{ vault_docker_registry_password }}' --dry-run=client -o=yaml | kubectl apply -f -" when: imagepullsecrets|length > 0 with_items: - "{{ bootstrap_namespace.split(',') }}"