From b2c4f1d90087ba8a5211c5e855e089664aee3b34 Mon Sep 17 00:00:00 2001 From: santhosh-tg <93243580+santhosh-tg@users.noreply.github.com> Date: Mon, 6 May 2024 14:37:57 +0530 Subject: [PATCH] [ED-3649] merge release-6.0.0 changes to 7.0.0 (#3998) * Fix syntax (#3894) * Updated vars for cron schedule (#3895) * Updated vars for cron schedule * Update main.yml * [ED-2793] Fix: release-6.0.0 installation issues (#3900) * Update Jenkins version * Fix job parameter * Fix error: You need to install jmespath prior to running json_query filter * Update main.yml with cassandra (#3845) (#3901) Co-authored-by: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> * ED-2838: ##Release-6.0.0 new variables (#3899) * ED-2838 Release-6.0.0 new variables * Added with comments * ED-2838 jenkins josb update and new variables * added storage endpoint detials * [ED-2793] Fix installation issues (#3905) * Update maven repo url * Add missing jobs * Fix docker creds secrets issue when having specialcharacters * [ED-2793] Add OCI specific vars (#3912) * Add OCI specific vars * Add mobile deeplink url var * Add ml_core_internal_access_token for private repo template Add ml_core_internal_access_token for private repo template * Update cassandra jenkins job config (#3927) * saving changes (#3935) Co-authored-by: ankitshahu * #OB-553 fix: Analytics service issue fixes for OCI * #OB-553 fix: Analytics service issue fixes for OCI * ED-3534:- Adoption Changes in ML service (#3963) * saving changes * added Upload api endpoint in public kong --------- Co-authored-by: ankitshahu * ED-3079: deploy nginx to support cors for s3 compatible object storage (#3915) * nginx-cors Signed-off-by: Deepak Devadathan * hostname changed Signed-off-by: Deepak Devadathan * test Signed-off-by: Deepak Devadathan * test Signed-off-by: Deepak Devadathan * added nginx-cors-public chart Signed-off-by: Deepak Devadathan * test Signed-off-by: Deepak Devadathan * corrected proxy conf Signed-off-by: Deepak Devadathan * corrected syntax Signed-off-by: Deepak Devadathan * changed nodeport Signed-off-by: Deepak Devadathan * removed duplicate Signed-off-by: Deepak Devadathan * nodeport change Signed-off-by: Deepak Devadathan * nodeport change Signed-off-by: Deepak Devadathan * changed public ip Signed-off-by: Deepak Devadathan * test Signed-off-by: Deepak Devadathan * removed nginx-cors ansible roles Signed-off-by: Deepak Devadathan * added jenkins job for nginx-cors-public deployment Signed-off-by: Deepak Devadathan * updated variable in values.j2 Signed-off-by: Deepak Devadathan * testing public ingress along with s3 cors Signed-off-by: Deepak Devadathan * testing with condition for csp Signed-off-by: Deepak Devadathan * removed nginx-cors-public Signed-off-by: Deepak Devadathan --------- Signed-off-by: Deepak Devadathan * Added new kafka topics in main.yml and config.j2 (#3970) * Issue #CO-824: dynamic redirect URL * Issue #CO-824: dynamic redirect URL * Issue #CO-824: SSO Isse condition update * Issue ED-0000 report service env update * Issue ED-0000 report service env update (#3977) * Issue ED-0000 report service env update * Issue ED-0000 report service env update (#3981) * ED-3534:- Adoption Changes in ML service (#3983) * helm chart for ml-analytics * Update and rename values.yaml to values.j2 * Update main.yml * Update ml-analytics-service.conf * Update ml-analytics-service.conf * Delete kubernetes/helm_charts/core/ml-analytics-service/templates/NOTES.txt * Update deployment.yaml * Update deployment.yaml * Update deployment.yaml * Update values.j2 * Update deployment.yaml * Update deployment.yaml * Update ml-analytics-service.conf * Update deployment.yaml * Update ml-analytics-service.conf * Update ml-analytics-service.conf * Update ml-analytics-service.conf * Update ml-analytics-service.conf * Updated ml-analytics-service.conf * added new kafka topics for observations * Update ml-analytics-service.conf --------- Co-authored-by: raghupathi Co-authored-by: Raghupathi Guduri * Issue #IQ-750 fix: added script for inquiry schema upload * Issue #IQ-750 fix: updated jenkins file * added the faust related changes * Issue #IQ-750 fix: updated jenkins script * added the value to ml_analytics_faust_applications_list * Update secrets.yml * ED-3534:- Adoption Changes in ML service (#3993) * updated druid ports * updated druid ports * key changes * key changes --------- Co-authored-by: vivek * Add release-6.0.0 missing vars * Add release-6.0.0 missing vars (#3995) * Add inquiry-schema-upload Jenkinsfile --------- Signed-off-by: Deepak Devadathan Co-authored-by: Raghupathi Guduri Co-authored-by: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Co-authored-by: Prasath Sivasubramaniyan Co-authored-by: Akash Shah Co-authored-by: Ankit Shahu Co-authored-by: ankitshahu Co-authored-by: sowmya-dixit Co-authored-by: Deepak Devadathan Co-authored-by: Prashant Gunjeti <153811259+prashanthShiksha@users.noreply.github.com> Co-authored-by: Shubham Bansal Co-authored-by: Rajeev Sathish Co-authored-by: Vivek M <125434153+Vivek-M-08@users.noreply.github.com> Co-authored-by: raghupathi Co-authored-by: Kumar Gauraw Co-authored-by: vivek --- .../login/resources/js/telemetry_service.js | 5 +- ansible/roles/kong-api/defaults/main.yml | 21 ++ .../ml-analytics-service/defaults/main.yml | 1 + .../ml-analytics-service/templates/config.j2 | 5 + ansible/roles/stack-sunbird/defaults/main.yml | 1 + .../templates/analytics_api_service.conf | 8 +- .../templates/ml-analytics-service.conf | 294 ++++++++++++++++++ .../templates/ml-core-service.env | 6 +- .../templates/ml-survey-service.env | 1 + .../templates/sunbird_report-service.env | 10 + .../core/analytics/templates/deployment.yaml | 16 +- .../helm_charts/core/analytics/values.j2 | 8 +- .../core/ml-analytics-service/.helmignore | 23 ++ .../core/ml-analytics-service/Chart.yaml | 24 ++ .../templates/_helpers.tpl | 62 ++++ .../templates/configmap.yaml | 11 + .../templates/deployment.yaml | 41 +++ .../templates/service.yaml | 14 + .../core/ml-analytics-service/values.j2 | 21 ++ .../templates/configMap.yaml | 4 + .../core/nginx-public-ingress/values.j2 | 69 ++++ .../upload/schema/inquiry/schema.Jenkinsfile | 55 ++++ .../ansible/inventory/dev/Core/common.yml | 49 ++- .../ansible/inventory/dev/Core/secrets.yml | 10 +- .../inventory/dev/DataPipeline/common.yml | 27 +- .../dev/KnowledgePlatform/common.yml | 34 +- 26 files changed, 781 insertions(+), 39 deletions(-) create mode 100644 ansible/roles/stack-sunbird/templates/ml-analytics-service.conf create mode 100644 kubernetes/helm_charts/core/ml-analytics-service/.helmignore create mode 100644 kubernetes/helm_charts/core/ml-analytics-service/Chart.yaml create mode 100644 kubernetes/helm_charts/core/ml-analytics-service/templates/_helpers.tpl create mode 100644 kubernetes/helm_charts/core/ml-analytics-service/templates/configmap.yaml create mode 100644 kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml create mode 100644 kubernetes/helm_charts/core/ml-analytics-service/templates/service.yaml create mode 100644 kubernetes/helm_charts/core/ml-analytics-service/values.j2 create mode 100644 pipelines/upload/schema/inquiry/schema.Jenkinsfile diff --git a/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js b/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js index 5b4b6c6125..1460374799 100644 --- a/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js +++ b/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js @@ -3159,6 +3159,7 @@ var handleGoogleAuthEvent = () => { redirectToLib(); } }; + var redirectToPortal = (redirectUrlPath) => { // redirectUrlPath for sso and self signUp const curUrlObj = window.location; var redirect_uri = getValueFromSession('redirect_uri'); @@ -3171,9 +3172,7 @@ var redirectToPortal = (redirectUrlPath) => { // redirectUrlPath for sso and sel const redirect_uriLocation = new URL(redirect_uri); if (client_id === 'android' || client_id === 'desktop') { window.location.href = sessionUrlObj.protocol + '//' + sessionUrlObj.host + redirectUrlPath + updatedQuery; - } else if(client_id === 'portal' && - redirectUrlPath === '/sign-in/sso/select-org' && - (redirect_uri.includes('dock.sunbirded.org') || redirect_uri.includes('dockstaging.sunbirded.org'))) { + } else if(client_id === 'portal' && redirectUrlPath === '/sign-in/sso/select-org' && (sessionUrlObj.host !== redirect_uriLocation.host)) { window.location.href = sessionUrlObj.protocol + '//' + sessionUrlObj.host + redirectUrlPath + updatedQuery; } else { window.location.href = redirect_uriLocation.protocol + '//' + redirect_uriLocation.host + diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index e3adfdaa64..64765e94c2 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -8300,6 +8300,27 @@ kong_apis: config.required: false config.enabled: false +- name: uploadFileToCloud + uris: "{{ cloud_service_prefix }}/mlcore/v1/files/upload" + upstream_url: "{{ ml_core_service_url }}/v1/cloud-services/files/upload" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - cloudUrlsCreate + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + - name: listPlatformPrograms uris: "{{ user_extension_prefix }}/mlcore/v1/programsByPlatformRoles" upstream_url: "{{ ml_core_service_url }}/v1/user-extension/programsByPlatformRoles" diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index a67906b1f8..85c826ac33 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -138,3 +138,4 @@ ml_analytics_batch_cron_hour: "{{ ml_batch_cron_hour | default('18') }}" ml_analytics_nvsk_cron_minute: "{{ ml_nvsk_cron_minute | default('30') }}" ml_analytics_nvsk_cron_hour: "{{ ml_nvsk_cron_hour | default('7') }}" ml_analytics_nvsk_cron_weekday: "{{ ml_nvsk_cron_weekday | default('4') }}" +ml_analytics_faust_applications_list: "'observations/py_observation_streaming observations/', 'observations/py_observation_evidence_streaming observations/', 'survey/py_survey_streaming survey/', 'survey/py_survey_evidence_streaming survey/'" diff --git a/ansible/roles/ml-analytics-service/templates/config.j2 b/ansible/roles/ml-analytics-service/templates/config.j2 index 935085d36a..201e9ab8b5 100644 --- a/ansible/roles/ml-analytics-service/templates/config.j2 +++ b/ansible/roles/ml-analytics-service/templates/config.j2 @@ -263,3 +263,8 @@ store = {{ ml_analytics_reports_store }} [ML_CORE_SERVICE_URL] url = {{ ml_analytics_core_service }} + + +[SERVICES] + +faust_applications_list = {{ ml_analytics_faust_applications_list }} diff --git a/ansible/roles/stack-sunbird/defaults/main.yml b/ansible/roles/stack-sunbird/defaults/main.yml index 835099cd9b..d66e2dae1d 100644 --- a/ansible/roles/stack-sunbird/defaults/main.yml +++ b/ansible/roles/stack-sunbird/defaults/main.yml @@ -264,6 +264,7 @@ service_env: ml-reports-service: ../../../../ansible/roles/stack-sunbird/templates/ml-reports-service.env ml-core-service: ../../../../ansible/roles/stack-sunbird/templates/ml-core-service.env ml-projects-service: ../../../../ansible/roles/stack-sunbird/templates/ml-projects-service.env + ml-analytics-service: ../../../../ansible/roles/stack-sunbird/templates/ml-analytics-service.conf inbound: ../../../../ansible/roles/stack-sunbird/templates/inbound.env outbound: - ../../../../ansible/roles/stack-sunbird/templates/outbound.env diff --git a/ansible/roles/stack-sunbird/templates/analytics_api_service.conf b/ansible/roles/stack-sunbird/templates/analytics_api_service.conf index 8bdcd63467..b9a8c838c8 100644 --- a/ansible/roles/stack-sunbird/templates/analytics_api_service.conf +++ b/ansible/roles/stack-sunbird/templates/analytics_api_service.conf @@ -201,10 +201,10 @@ public { } } cloud_storage_type="{{cloud_service_provider}}" -storage.key.config="{{cloud_public_storage_accountname}}" -storage.secret.config="{{cloud_public_storage_secret}}" -public.storage.key.config="{{cloud_public_storage_accountname}}" -public.storage.secret.config="{{cloud_public_storage_secret}}" +storage.key.config="cloud_storage_key" +storage.secret.config="cloud_storage_secret" +public.storage.key.config="public_cloud_storage_key" +public.storage.secret.config="public_cloud_storage_secret" cloud_storage_endpoint_with_protocol="{{cloud_storage_url}}" metrics.time.interval.min=30 cache.refresh.time.interval.min=1440 diff --git a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf new file mode 100644 index 0000000000..707dc83e6f --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf @@ -0,0 +1,294 @@ +[ML_APP_NAME] + +survey_app = {{ ml_survey_app_name | default('sunbirdsurvey') }} + +integrated_app = {{ ml_integrated_app_name | default('sunbird') }} + +integrated_portal = {{ ml_integrated_portal | default('dev.sunbird.portal') }} + + +[API_ENDPOINTS] + +base_url = https://{{ domain_name }}/ + + +[API_HEADERS] + +content_type = application/json + +authorization = {{ml_api_auth_token | default('sunbird_api_auth_token')}} + +internal_access_token = {{ml_api_access_token | default('ml_core_internal_access_token')}} + +authorization_access_token = {{ ml_analytics_authorization_access_token }} + +[ML_SURVEY_SERVICE_URL] + +url = http://{{private_ingressgateway_ip}}/ml-survey + +entity_related_end_point = assessment/api/v1/entities/relatedEntities/ + +user_profile_end_point = assessment/api/v1/userExtension/getProfile/ + +evidence_base_url = {{ cloud_storage_url }}/{{ ml_analytics_public_storage }}/ + + +[MONGO] + +url = mongodb://{{ml_mongodb_host | default(groups['mongo_master'][0]+':27017')}} + +database_name = {{ml_mongodb | default('ml-survey')}} + + +# ------ Mongo Collections ------- # +observation_sub_collection = observationSubmissions + +solutions_collection = solutions + +observations_collection = observations + +entity_type_collection = entityTypes + +questions_collection = questions + +criteria_collection = criteria + +entities_collection = entities + +programs_collection = programs + +user_roles_collection = userRoles + +criteria_questions_collection = criteriaQuestions + +projects_collection = projects + +survey_submissions_collection = surveySubmissions + +survey_collection = surveys + +[DRUID] + +metadata_url = http://{{groups['druid'][0]}}:8081/druid/coordinator/v1/datasources/ + +batch_url = http://{{groups['druid'][0]}}:8081/druid/indexer/v1/task + +metadata_rollup_url = http://{{groups['druid'][0]}}:8081/druid/coordinator/v1/datasources/ + +batch_rollup_url = http://{{groups['druid'][0]}}:8081/druid/indexer/v1/task + +urlQuery = http://{{groups['druid'][0]}}:8082/druid/v2?pretty + +datasource_url = http://{{groups['ml-raw-broker'][0]}}:{{ml_raw_broker_port}}/druid/v2/datasources + +sql_url = http://{{groups['ml-raw-broker'][0]}}:{{ml_raw_broker_port}}/druid/v2/sql/ + +coordinator_url = http://{{groups['ml-raw-coordinator'][0]}}:{{ml_raw_coordinator_port}}/status/health + +overload_url = http://{{groups['ml-raw-overload'][0]}}:{{ml_raw_overload_port}}/status/health + +historical_url = http://{{groups['ml-raw-historical'][0]}}:{{ml_raw_historical_port}}/status/health + +observation_status_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_code","school_externalId","state_name","state_code","state_externalId","district_name","district_code","district_externalId","block_name","block_code","block_externalId","cluster_name","cluster_code","cluster_externalId","completedDate","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name","ecm_marked_na","board_name","updatedAt","organisation_id","user_type","observed_school_name","observed_school_id","observed_school_code","observed_state_name","observed_state_id","observed_state_code","observed_district_name","observed_district_id","observed_district_code","observed_block_name","observed_block_id","observed_block_code","observed_cluster_name","observed_cluster_id","observed_cluster_code","isRubricDriven","criteriaLevelReport"]},"metricsSpec":[]}}}' + +project_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-project","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"auto"},"dimensionsSpec":{"dimensions":[]},"metricsSpec":[]}}}' + +ml_distinctCnt_obs_status_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' + +ml_distinctCnt_obs_domain_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' + +ml_distinctCnt_obs_domain_criteria_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain-criteria","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"string","name":"criteria_name"},{"type":"string","name":"criteria_score"},{"type":"string","name":"criteria_id"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' + +ml_distinctCnt_projects_status_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"project_title"},{"type":"string","name":"solution_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"long","name":"no_of_imp_with_evidence_inprogress"},{"type":"string","name":"time_stamp"},{"type":"long","name":"no_of_certificate_issued"}]},"metricsSpec":[]}}}' + +ml_distinctCnt_prglevel_projects_status_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-programLevel-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"},{"type":"long","name":"no_of_certificate_issued"}]},"metricsSpec":[]}}}' + +ml_distinctCnt_survey_status_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-surveydistinctCount-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"survey_name"},{"type":"string","name":"survey_id"},{"type":"string","name":"submission_status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_submissions"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' + +observation_status_rollup_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-observation-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["status","user_id","solution_id","submission_id","entity_name","completedDate","program_id","private_program","solution_type","updatedAt","role_title","solution_name","program_name","channel","parent_channel","block_name","district_name","school_name","cluster_name","state_name","organisation_name","board_name","district_externalId","state_externalId","block_externalId","cluster_externalId","school_externalId","organisation_id",{"type":"long","name":"status_code"}]},"metricsSpec":[{"type":"count","name":"count"},{"type":"longSum","name":"sum___v","fieldName":"__v","expression":null},{"type":"HLLSketchBuild","name":"count_distinct_solution","fieldName":"solution_id","lgK":12,"tgtHllType":"HLL_4","round":false},{"type":"HLLSketchBuild","name":"count_distinct_submission_id","fieldName":"submission_id","lgK":12,"tgtHllType":"HLL_4","round":false},{"type":"HLLSketchBuild","name":"count_distinct_user_id","fieldName":"user_id","lgK":12,"tgtHllType":"HLL_4","round":false}]}}}' + +project_rollup_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"iso"},"dimensionsSpec":{"dimensions":["project_title","project_goal","area_of_improvement","status_of_project","tasks_name","tasks_status","designation","task_evidence_status","project_id","task_id","project_created_type","parent_channel","program_id","program_name","project_updated_date","createdBy","program_externalId","private_program","task_deleted_flag","project_terms_and_condition","state_externalId","block_externalId","district_externalId","cluster_externalId","school_externalId","state_name","block_name","district_name","cluster_name","school_name","board_name","organisation_name","solution_id","organisation_id",{"name":"status_code","type":"long"}]},"metricsSpec":[{"name":"count","type":"count"},{"name":"sum___v","type":"longSum","fieldName":"__v"},{"name":"sum_status_code","type":"longMax","fieldName":"status_code"},{"type":"HLLSketchBuild","name":"count_of_createBy","fieldName":"createdBy"},{"type":"HLLSketchBuild","name":"count_of_project_id","fieldName":"project_id"},{"type":"HLLSketchBuild","name":"count_of_solution_id","fieldName":"solution_id"},{"type":"HLLSketchBuild","name":"count_of_program_id","fieldName":"program_id"}]}}}' + +ml_survey_rollup_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel",{"type":"long","name":"status_code"}, "solution_name", "solution_id"]},"metricsSpec":[{"name":"count","type":"count"},{"name":"sum___v","type":"longSum","fieldName":"__v"},{"name":"sum_status_code","type":"longMax","fieldName":"status_code"},{"type":"HLLSketchBuild","name":"count_of_user_id","fieldName":"user_id"},{"type":"HLLSketchBuild","name":"count_of_survey_submission_id","fieldName":"survey_submission_id"},{"type":"HLLSketchBuild","name":"count_of_solution_id","fieldName":"solution_id"},{"type":"HLLSketchBuild","name":"count_of_program_id","fieldName":"program_id"}]}}}' + +survey_status_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel", "solution_name", "solution_id","private_program","state_code","school_code","district_code","block_code","cluster_code"]},"metricsSpec":[]}}}' + +observation_query_spec = '{"queryType":"scan","dataSource":"sl-observation","resultFormat":"list","columns":["completedDate","createdAt","createdBy","criteriaExternalId","criteriaId","criteriaName","entityType","entityTypeId","observationId","observationName","observationSubmissionId","questionAnswer","questionECM","questionExternalId","questionId","questionName","questionResponseLabel","questionResponseType","solutionExternalId","solutionId","solutionName","updatedAt","instanceParentId","instanceId","instanceParentResponsetype","instanceParentQuestion","questionSequenceByEcm","maxScore","minScore","percentageScore","pointsBasedScoreInParent","totalScore","scoreAchieved","totalpercentage","instanceParentExternalId","instanceParentEcmSequence","remarks","total_evidences","evidence_count","school","block","district","cluster","state","schoolName","blockName","districtName","clusterName","stateName","schoolExternalId","blockExternalId","districtExternalId","clusterExternalId","stateExternalId","schoolTypes","administrationTypes","instanceParentCriteriaId","instanceParentCriteriaExternalId","instanceParentCriteriaName","role_title","location_validated_with_geotag","distance_in_meters","entity","entityExternalId","entityName","isAPrivateProgram","programId","programName","programExternalId","questionResponseLabel_number","criteriaLevel","criteriaScore","submissionNumber","submissionTitle","channel","parent_channel","user_districtName","user_blockName","user_clusterName","appName","evidences","user_stateName","domainName","domainExternalId","childName","childType","childExternalid","level","criteriaDescription","programDescription","solutionDescription","label","imp_project_id","imp_project_title","imp_project_goal","imp_project_externalId","ancestorName","scoringSystem","domainLevel","domainScore","criteriaLevelReport","user_schoolName","user_schoolId","user_schoolUDISE_code","solution_type","organisation_name","user_boardName","district_externalId","state_externalId","block_externalId","cluster_externalId","organisation_id","user_type"],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]}' + +observation_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"entityType"},{"type":"string","name":"entityTypeId"},{"type":"string","name":"observationId"},{"type":"string","name":"observationName"},{"type":"string","name":"observationSubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"school"},{"type":"string","name":"block"},{"type":"string","name":"district"},{"type":"string","name":"cluster"},{"type":"string","name":"state"},{"type":"string","name":"schoolName"},{"type":"string","name":"blockName"},{"type":"string","name":"districtName"},{"type":"string","name":"clusterName"},{"type":"string","name":"stateName"},{"type":"string","name":"schoolExternalId"},{"type":"string","name":"blockExternalId"},{"type":"string","name":"districtExternalId"},{"type":"string","name":"clusterExternalId"},{"type":"string","name":"stateExternalId"},{"type":"string","name":"schoolTypes"},{"type":"string","name":"administrationTypes"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"role_title"},{"type":"string","name":"location_validated_with_geotag"},{"type":"string","name":"distance_in_meters"},{"type":"string","name":"entity"},{"type":"string","name":"entityExternalId"},{"type":"string","name":"entityName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"criteriaLevel"},{"type":"string","name":"criteriaScore"},{"type":"string","name":"submissionNumber"},{"type":"string","name":"submissionTitle"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"user_districtName"},{"type":"string","name":"user_blockName"},{"type":"string","name":"user_clusterName"},{"type":"string","name":"appName"},{"type":"string","name":"evidences"},{"type":"string","name":"user_stateName"},{"type":"string","name":"domainName"},{"type":"string","name":"domainExternalId"},{"type":"string","name":"childName"},{"type":"string","name":"childType"},{"type":"string","name":"childExternalid"},{"type":"string","name":"level"},{"type":"string","name":"criteriaDescription"},{"type":"string","name":"programDescription"},{"type":"string","name":"solutionDescription"},{"type":"string","name":"label"},{"type":"string","name":"imp_project_id"},{"type":"string","name":"imp_project_title"},{"type":"string","name":"imp_project_goal"},{"type":"string","name":"imp_project_externalId"},{"type":"string","name":"ancestorName"},{"type":"string","name":"scoringSystem"},{"type":"string","name":"domainLevel"},{"type":"string","name":"domainScore"},{"name":"criteriaLevelReport","type":"boolean"},{"type":"string","name":"user_schoolName"},{"type":"string","name":"user_schoolId"},{"type":"string","name":"user_schoolUDISE_code"},{"type":"string","name":"solution_type"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_boardName"},{"type":"string","name":"district_externalId"},{"type":"string","name":"state_externalId"},{"type":"string","name":"block_externalId"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"user_type"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}' + +survey_query_spec = '{"queryType":"scan","dataSource":"sl-survey","resultFormat":"list","columns":["completedDate","createdAt","createdBy","criteriaExternalId","criteriaId","criteriaName","surveyId","surveyName","surveySubmissionId","questionAnswer","questionECM","questionExternalId","questionId","questionName","questionResponseLabel","questionResponseType","solutionExternalId","solutionId","solutionName","updatedAt","instanceParentId","instanceId","instanceParentResponsetype","instanceParentQuestion","questionSequenceByEcm","maxScore","minScore","percentageScore","pointsBasedScoreInParent","totalScore","scoreAchieved","totalpercentage","instanceParentExternalId","instanceParentEcmSequence","remarks","total_evidences","evidence_count","instanceParentCriteriaId","instanceParentCriteriaExternalId","instanceParentCriteriaName","isAPrivateProgram","programId","programName","programExternalId","questionResponseLabel_number","channel","parent_channel","appName","organisation_name","user_subtype","user_type","board_name","district_code","district_name","district_externalId","block_code","block_name","block_externalId","school_code","school_name","school_externalId","cluster_code","cluster_name","cluster_externalId","state_code","state_name","state_externalId","organisation_id","evidences"],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]}' + +survey_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-survey","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"surveyId"},{"type":"string","name":"surveyName"},{"type":"string","name":"surveySubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"evidences"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"appName"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_subtype"},{"type":"string","name":"user_type"},{"type":"string","name":"board_name"},{"type":"string","name":"district_code"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_code"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"school_code"},{"type":"string","name":"school_name"},{"type":"string","name":"school_externalId"},{"type":"string","name":"cluster_code"},{"type":"string","name":"cluster_name"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"state_code"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}' + +intervals = '["1901-01-01T00:00:00+00:00/2020-01-01T00:00:00+00:00","2020-01-01T00:00:00+00:00/2020-06-01T00:00:00+00:00","2020-06-01T00:00:00+00:00/2021-01-01T00:00:00+00:00","2021-01-01T00:00:00+00:00/2021-06-01T00:00:00+00:00","2021-06-01T00:00:00+00:00/2022-01-01T00:00:00+00:00","2022-01-01T00:00:00+00:00/2022-03-01T00:00:00+00:00","2022-03-01T00:00:00+00:00/2022-06-01T00:00:00+00:00","2022-06-01T00:00:00+00:00/2022-09-01T00:00:00+00:00","2022-09-01T00:00:00+00:00/2023-01-01T00:00:00+00:00"]' + + +[KAFKA] + +url = {{ml_kafka_host | default(groups['processing-cluster-kafka'][0]+':9092')}} + +observation_raw_topic = {{ env_name }}.ml.observation.raw + +observation_druid_topic = {{ env_name }}.ml.observation.druid + +observation_evidence_druid_topic = {{ env_name }}.ml.observation.evidence.druid + +survey_evidence_druid_topic = {{ env_name }}.ml.survey.evidence.druid + +survey_raw_topic = {{ env_name }}.ml.survey.raw + +survey_druid_topic = {{ env_name }}.ml.survey.druid + +survey_meta_druid_topic = {{ env_name }}.ml.survey.meta + +survey_completed_druid_topic = {{ env_name }}.ml.survey.status.completed + +survey_started_druid_topic = {{ env_name }}.ml.survey.status.started + +survey_inprogress_druid_topic = {{ env_name }}.ml.survey.status.inprogress + +observation_meta_druid_topic = {{ env_name }}.ml.observation.meta + +observation_started_druid_topic = {{ env_name }}.ml.observation.status.started + +observation_inprogress_druid_topic = {{ env_name }}.ml.observation.status.inprogress + +observation_completed_druid_topic = {{ env_name }}.ml.observation.status.completed + +[LOGS] + +observation_streaming_success_error = /opt/sparkjobs/ml-analytics-service/logs/observation/ + +observation_streaming_evidence_success_error = /opt/sparkjobs/ml-analytics-service/logs/observation/evidence/ + +observation_status_success_error = /opt/sparkjobs/ml-analytics-service/logs/observation/status/ + +project_success_error = /opt/sparkjobs/ml-analytics-service/logs/project/ + +survey_evidence_streaming_success_error = /opt/sparkjobs/ml-analytics-service/logs/project/evidence/ + +survey_streaming_success_error = /opt/sparkjobs/ml-analytics-service/logs/survey/ + +survey_status_success_error = /opt/sparkjobs/ml-analytics-service/logs/survey/status/ + + +[CLOUD] + +container_name = {{ cloud_storage_telemetry_bucketname }} + +[OUTPUT_DIR] + +project = /opt/sparkjobs/source/projects/output + +observation_status = /opt/sparkjobs/source/observations/status/output + +observation_distinctCount_status = /opt/sparkjobs/source/observations/distinctCount/output + +observation_distinctCount_domain = /opt/sparkjobs/source/observations/distinctCount_domain/output + +observation_distinctCount_domain_criteria = /opt/sparkjobs/source/observations/distinctCount_domain_criteria/output + +projects_distinctCount = /opt/sparkjobs/source/projects/distinctCount/output + +projects_distinctCount_prgmlevel = /opt/sparkjobs/source/projects/distinctCountPrglevel/output + +survey_distinctCount_status = /opt/sparkjobs/source/survey/distinctCount/output + +project_rollup = /opt/sparkjobs/source/projects/output_rollup + +observation_status_rollup = /opt/sparkjobs/source/observations/output_rollup + +survey_rollup = /opt/sparkjobs/source/survey/output + +survey_status = /opt/sparkjobs/source/survey/status/output + +observation_sub_ids = /opt/sparkjobs/ml-analytics-service/observations/submissions.csv + +observation_druid_data = /opt/sparkjobs/source/observations/ + +survey_sub_ids = /opt/sparkjobs/ml-analytics-service/survey/submissions.csv + +survey_druid_data = /opt/sparkjobs/source/survey/ + +program_text_file = /opt/sparkjobs/ml-analytics-service/projects/program_ids.txt + +[SLACK] + +token = {{ ml_analytics_slack_token | default('') }} + +channel = {{ ml_analytics_slack_channel | default('') }} + + +[VAM] + +druid_query_url = {{ ml_druid_query | default('') }} + +program_dashboard_data = {{ ml_program_data | default('') }} + + +[COMMON] + +cloud_module_path = /opt/sparkjobs/ml-analytics-service/cloud_storage + +observation_blob_path = observation/status/ + +projects_blob_path = projects/ + +observation_distinctCount_blob_path = observation/distinctCount/ + +observation_distinctCount_domain_blob_path = observation/distinctCount_domain/ + +observation_distinctCount_domain_criteria_blob_path = observation/distinctCount_domain_criteria/ + +projects_distinctCnt_blob_path = projects/distinctCount/ + +projects_distinctCnt_prgmlevel_blob_path = projects/distinctCountPrglevel/ + +survey_distinctCount_blob_path = survey/distinctCount/ + +projects_rollup_blob_path = projects/rollup + +observation_rollup_blob_path = observation/rollup + +survey_rollup_blob_path = survey/rollup/ + +survey_blob_path = survey/status/ + +projects_program_csv = {{ ml_program_blob_path | default('') }} + +observation_batch_ingestion_data_del = observation/batchDeletion + +survey_batch_ingestion_data_del = survey/batchDeletion + +cname_url = {{ cloud_storage_url }}/{{ cloud_storage_samiksha_bucketname }} + +nvsk_imp_projects_data_local_path = /opt/sparkjobs/ml-analytics-service/urgent_data_metrics/output/ + +nvsk_imp_projects_data_blob_path = Manage_Learn_Data/micro_improvement/ + +[API_CREDENTIALS] + +client_id = {{ ml_analytics_client_id }} + +client_secret = {{ ml_analytics_client_secret }} + +username = {{ ml_analytics_username }} + +password = {{ ml_analytics_password }} + +[JSON_VARIABLE] + +createdBy = {{ ml_analytics_createdBy }} + +container = {{ ml_analytics_reports_container }} + +store = {{ ml_analytics_reports_store }} + +[ML_CORE_SERVICE_URL] + +url = http://{{private_ingressgateway_ip}}/ml-core + +[SERVICES] + +faust_applications_list = {{ faust_applications_list }} diff --git a/ansible/roles/stack-sunbird/templates/ml-core-service.env b/ansible/roles/stack-sunbird/templates/ml-core-service.env index 7a8b3cbf55..6ffb7a666c 100755 --- a/ansible/roles/stack-sunbird/templates/ml-core-service.env +++ b/ansible/roles/stack-sunbird/templates/ml-core-service.env @@ -87,4 +87,8 @@ ID={{ml_core_service_id | default(env_name+"."+sunbird_instance+".ml.core.servic TELEMETRY_ON_OFF={{manage_learn_telemetry_on_off | default("ON")}} #Topic name for telemetry -TELEMETRY_TOPIC={{telemetry_raw_topic_name | default(env_name+".telemetry.raw")}} \ No newline at end of file +TELEMETRY_TOPIC={{telemetry_raw_topic_name | default(env_name+".telemetry.raw")}} + +#Public base url of sunbird +PUBLIC_BASE_URL={{ml_core_public_base_url | default("")}} + diff --git a/ansible/roles/stack-sunbird/templates/ml-survey-service.env b/ansible/roles/stack-sunbird/templates/ml-survey-service.env index 277da9b836..fa9511083a 100644 --- a/ansible/roles/stack-sunbird/templates/ml-survey-service.env +++ b/ansible/roles/stack-sunbird/templates/ml-survey-service.env @@ -86,3 +86,4 @@ TELEMETRY_ON_OFF={{manage_learn_telemetry_on_off | default("ON")}} #Topic name for telemetry TELEMETRY_TOPIC={{telemetry_raw_topic_name | default(env_name+".telemetry.raw")}} + diff --git a/ansible/roles/stack-sunbird/templates/sunbird_report-service.env b/ansible/roles/stack-sunbird/templates/sunbird_report-service.env index a2ebfce02a..b13358fcac 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_report-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_report-service.env @@ -29,3 +29,13 @@ sunbird_cache_ttl={{sunbird_cache_ttl}} sunbird_report_sas_expiry_in_minutes=3600 sunbird_dataservice_url={{sunbird_dataservice_url}} +# 6.0.0 + +sunbird_cloud_storage_provider={{cloud_service_provider}} +cloud_private_storage_accountname={{cloud_private_storage_accountname | default("")}} +cloud_private_storage_secret={{cloud_private_storage_secret | default("")}} +cloud_storage_privatereports_bucketname={{cloud_storage_privatereports_bucketname | default("reports")}} +cloud_storage_resourceBundle_bucketname={{cloud_storage_label_bucketname | default("label")}} +cloud_private_storage_region={{cloud_private_storage_region | default("ap-south-1")}} +cloud_private_storage_project={{cloud_private_storage_project | default("")}} +cloud_private_storage_endpoint={{cloud_private_storage_endpoint | default("")}} diff --git a/kubernetes/helm_charts/core/analytics/templates/deployment.yaml b/kubernetes/helm_charts/core/analytics/templates/deployment.yaml index 0926360f76..57198cb77b 100644 --- a/kubernetes/helm_charts/core/analytics/templates/deployment.yaml +++ b/kubernetes/helm_charts/core/analytics/templates/deployment.yaml @@ -35,14 +35,14 @@ spec: value: {{ .Values.env.min_heap | quote }} - name: MAX_HEAP value: {{ .Values.env.max_heap | quote }} - - name: azure_storage_secret - value: {{ .Values.env.azure_private_account_secret | quote }} - - name: azure_storage_key - value: {{ .Values.env.azure_private_account_name | quote }} - - name: public_azure_storage_secret - value: {{ .Values.env.azure_public_account_secret | quote }} - - name: public_azure_storage_key - value: {{ .Values.env.azure_public_account_name | quote }} + - name: cloud_storage_secret + value: {{ .Values.env.cloud_private_account_secret | quote }} + - name: cloud_storage_key + value: {{ .Values.env.cloud_private_account_name | quote }} + - name: public_cloud_storage_secret + value: {{ .Values.env.cloud_public_account_secret | quote }} + - name: public_cloud_storage_key + value: {{ .Values.env.cloud_public_account_name | quote }} - name: _JAVA_OPTIONS value: -Dlog4j2.formatMsgNoLookups=true envFrom: diff --git a/kubernetes/helm_charts/core/analytics/values.j2 b/kubernetes/helm_charts/core/analytics/values.j2 index 277f840947..5aae4e1daa 100644 --- a/kubernetes/helm_charts/core/analytics/values.j2 +++ b/kubernetes/helm_charts/core/analytics/values.j2 @@ -8,10 +8,10 @@ env: javaoptions: {{analytics_java_mem_limit|default('-Xmx600m')}} min_heap: {{analytics_min_heap_limit|default('-Xms1g')}} max_heap: {{analytics_max_heap_limit|default('-Xmx2g')}} - azure_private_account_secret: {{ cloud_private_storage_secret }} - azure_private_account_name: {{ cloud_private_storage_accountname }} - azure_public_account_secret: {{ cloud_public_storage_secret }} - azure_public_account_name: {{ cloud_public_storage_accountname }} + cloud_private_account_secret: {{ cloud_private_storage_secret }} + cloud_private_account_name: {{ cloud_private_storage_accountname }} + cloud_public_account_secret: {{ cloud_public_storage_secret }} + cloud_public_account_name: {{ cloud_public_storage_accountname }} replicaCount: {{analytics_replicacount|default(1)}} repository: {{analytics_repository|default('sunbird-analytics-service')}} image_tag: {{ image_tag }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/.helmignore b/kubernetes/helm_charts/core/ml-analytics-service/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/core/ml-analytics-service/Chart.yaml b/kubernetes/helm_charts/core/ml-analytics-service/Chart.yaml new file mode 100644 index 0000000000..eb2395299a --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: ml-analytics-service +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/_helpers.tpl b/kubernetes/helm_charts/core/ml-analytics-service/templates/_helpers.tpl new file mode 100644 index 0000000000..3f238f7ff7 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "ml-analytics-service.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ml-analytics-service.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ml-analytics-service.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "ml-analytics-service.labels" -}} +helm.sh/chart: {{ include "ml-analytics-service.chart" . }} +{{ include "ml-analytics-service.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "ml-analytics-service.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ml-analytics-service.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ml-analytics-service.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "ml-analytics-service.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/configmap.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/configmap.yaml new file mode 100644 index 0000000000..5f03f92832 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/configmap.yaml @@ -0,0 +1,11 @@ +--- +{{- $keys := .Files.Glob "keys/*" }} +{{ if $keys }} +apiVersion: v1 +kind: Secret +metadata: + name: ml-analytics-access-keys +type: Opaque +data: +{{ (.Files.Glob "keys/*").AsSecrets | indent 2 }} +{{ end }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml new file mode 100644 index 0000000000..d28628b7e5 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + volumes: + - name: {{ .Chart.Name }}-config + configMap: + name: {{ .Chart.Name }}-config + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + volumeMounts: + - name: {{ .Chart.Name }}-config + mountPath: /opt/sparkjobs/ml-analytics-service/config.ini + subPath: ml-analytics-service.conf diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/service.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/service.yaml new file mode 100644 index 0000000000..4b47ed5332 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/values.j2 b/kubernetes/helm_charts/core/ml-analytics-service/values.j2 new file mode 100644 index 0000000000..2a2a1c8638 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/values.j2 @@ -0,0 +1,21 @@ +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{ml_analytics_replicacount|default(1)}} +repository: {{ml_analytics_repository|default('ml-analytics-service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{ml_analytics_cpu_req|default('1000m')}} + memory: {{ml_analytics_mem_req|default('1000Mi')}} + limits: + cpu: {{ml_analytics_cpu_limit|default('2')}} + memory: {{ml_analytics_mem_limit|default('2048Mi')}} +network: + port: 7000 + targetport: 7000 +strategy: + type: RollingUpdate + maxsurge: {{ ml_core_maxsurge|default('25%') }} + maxunavailable: {{ ml_core_maxunavailable|default('25%') }} diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/templates/configMap.yaml b/kubernetes/helm_charts/core/nginx-public-ingress/templates/configMap.yaml index 0f7f0dcc16..3a04ccb80c 100644 --- a/kubernetes/helm_charts/core/nginx-public-ingress/templates/configMap.yaml +++ b/kubernetes/helm_charts/core/nginx-public-ingress/templates/configMap.yaml @@ -6,6 +6,10 @@ metadata: data: proxy-default.conf: | {{ .Values.proxyconfig | indent 4 }} +{{- if eq .Values.csp "oci" }} + cors-proxy-default.conf: | +{{ .Values.corsproxyconfig | indent 4 }} +{{- end }} compression.conf: | {{ .Values.compressionConfig | indent 4 }} diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 b/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 index 8d5ce97115..24c223cb8f 100644 --- a/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 +++ b/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 @@ -1,6 +1,7 @@ #jinja2:lstrip_blocks: True namespace: {{ namespace }} +csp: {{cloud_service_provider}} merge_domain_status: {{ merge_domain_status | lower }} service: annotations: {{nginx_public_ingress_service_annotations | d('') | to_json}} @@ -64,6 +65,74 @@ resources: repository: {{proxy_repository|default('proxy')}} image_tag: {{ image_tag }} +corsproxyconfig: |- + {% if proto=='https' %} + server { + if ($host = files.{{domain_name}}) { + return 301 https://$host$request_uri; + } + listen 80 ; + listen [::]:80 ; + server_name files.{{domain_name}}; + return 404; + } + {% endif %} + server { + {% if proto=='http' %} + listen 80; + listen [::]:80; + {% else %} + listen [::]:443 ssl ipv6only=on; + listen 443 ssl; + ssl_certificate /etc/secrets/site.crt; + ssl_certificate_key /etc/secrets/site.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA HIGH !RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS"; + {% endif %} + server_name files.{{domain_name}}; + client_max_body_size 0; + root /var/www/html; + resolver {{ kube_dns_ip }} valid=30s; + + location / { + # handle cors and allow all + if ($request_method = OPTIONS ) { + add_header Access-Control-Allow-Origin *; + add_header Access-Control-Allow-Methods "GET, OPTIONS, PATCH, POST, PUT, HEAD"; + add_header Access-Control-Allow-Headers "Access-Control-Allow-Origin, Authorization, Content-Type, user-id, Accept, Accept-Encoding, Accept-Language, Access-Control-Request-Headers, Access-Control-Request-Method, Cache-Control, DNT, User-Agent, X-Amz-Algorithm, X-Amz-Credential, X-Amz-Date, Amz-Expires, X-Amz-SignedHeaders, X-Amz-Signature, x-ms-blob-type"; + add_header Access-Control-Allow-Credentials "true"; + add_header Content-Length 0; + add_header Content-Type text/plain; + return 204; + } + + proxy_set_header Host "{{ cloud_storage_url | replace('https://', '') }}"; + # remove any CORS header from backend OSS S3 + proxy_hide_header Access-Control-Allow-Origin; + proxy_hide_header Access-Control-Allow-Methods; + proxy_hide_header Access-Control-Allow-Headers; + proxy_hide_header Access-Control-Allow-Credentials; + + # inject our own CORS header to allow what we wanted + add_header Access-Control-Allow-Credentials "true" always; + add_header Access-Control-Expose-Headers 'Content-Length,Content-Range,Connection,opc-client-info,opc-request-id' always; + add_header Access-Control-Allow-Origin * always; + add_header Access-Control-Allow-Methods "GET,OPTIONS,PATCH,POST,PUT,HEAD" always; + add_header Access-Control-Allow-Headers "Access-Control-Allow-Origin, Authorization, Content-Type, user-id, Accept,Accept-Encoding,Accept-Language, Access-Control-Request-Headers, Access-Control-Request-Method,Cache-Control,DNT,Host,Origin,Pragma,Referer,User-Agent, X-Amz-Algorithm, X-Amz-Credential, X-Amz-Date, Amz-Expires, X-Amz-SignedHeaders, X-Amz-Signature, x-ms-blob-type" always; + # + add_header Referer ""; + proxy_pass {{cloud_storage_url}}; + + # if get request, trim the query string + if ($request_method = GET ) { + proxy_pass {{cloud_storage_url}}$uri; + } + + + } + } + + proxyconfig: |- {% if proto=='https' %} server { diff --git a/pipelines/upload/schema/inquiry/schema.Jenkinsfile b/pipelines/upload/schema/inquiry/schema.Jenkinsfile new file mode 100644 index 0000000000..b9020d0c6e --- /dev/null +++ b/pipelines/upload/schema/inquiry/schema.Jenkinsfile @@ -0,0 +1,55 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + } + + ansiColor('xterm') { + values = [:] + currentWs = sh(returnStdout: true, script: 'pwd').trim() + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + stage('deploy artifact'){ + sh """ + rm -rf schema_repo + git clone ${params.schema_repo} -b ${params.schema_repo_branch_or_tag} schema_repo + """ + ansiblePlaybook = "${currentWs}/ansible/kp_upload-schema.yml" + ansibleExtraArgs = "--extra-vars \" source_name=${params.source_folder} source_path=${currentWs}/schema_repo/schemas \" --vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = 'SUCCESS' + currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}, schema_repo_branch_or_tag: ${params.schema_repo_branch_or_tag}" + } + } + summary() + } + catch (err) { + currentBuild.result = 'FAILURE' + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} diff --git a/private_repo/ansible/inventory/dev/Core/common.yml b/private_repo/ansible/inventory/dev/Core/common.yml index da2d8a9694..fe1f96338a 100644 --- a/private_repo/ansible/inventory/dev/Core/common.yml +++ b/private_repo/ansible/inventory/dev/Core/common.yml @@ -10,21 +10,25 @@ private_ingressgateway_ip: "" # your private kubernetes load balancer ip # Cloud Service Provider Variables # If cloud_service_provider is AWS then update with access key as value # Example: cloud_public_storage_accountname: "AKSHKSJHFJDHJDSHFKSD" +# If cloud_service_provider is OCI then update with access key as value +# Example: cloud_public_storage_accountname: "7855461d26e1d8f211a04ec59cfd3037c8d1c6dd" # If cloud_service_provider is gloud(GCP) then update with service account name # Example: cloud_public_storage_accountname: "cloudstorage-gcp-test.iam.gserviceaccount.com" # If cloud_service_provider is AZURE then update with stoage account name # Example: cloud_public_storage_accountname: "azurestotageaccount" cloud_public_storage_accountname: "" -# If cloud_service_provider is AWS then update with region +# If cloud_service_provider is AWS/OCI then update with region # Example: cloud_public_storage_region: us-east-1 cloud_public_storage_region: "" # If cloud_service_provider is gcp then update this variable with project id # Example: cloud_public_storage_project: "sunbird-gcp-test" cloud_public_storage_project: "" - +# If cloud_service_provider is OCI then update this variable with namespace value +# Example: cloud_public_storage_namespace: "apsyjjclx" +cloud_public_storage_namespace: "" # Create object storage for each below mentioned variables and update accordingly -# If cloud_service_provider is AWS update with bucket name +# If cloud_service_provider is AWS/OCI update with bucket name # If cloud_service_provider is gcloud(GCP) update with bucket name # If cloud_service_provider is AZURE update with container name # Example: cloud_storage_certqr_bucketname: "certqr-storage" @@ -96,6 +100,8 @@ cloud_storage_management_bucketname: "" # cloud_storage_url: https://storage.googleapis.com # AWS # cloud_storage_url: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" +# OCI +# cloud_storage_url: "https://{{ cloud_public_storage_namespace }}.compat.objectstorage.{{ cloud_public_storage_region }}.oraclecloud.com" # Azure cloud_storage_url: "https://{{ cloud_public_storage_accountname }}.blob.core.windows.net" @@ -106,6 +112,18 @@ cloud_storage_url: "https://{{ cloud_public_storage_accountname }}.blob.core.win # for google: # nginx_private_ingress_annotations: # cloud.google.com/load-balancer-type: "Internal" +# for OCI private +#nginx_private_ingress_annotations: +# service.beta.kubernetes.io/oci-load-balancer-internal: "true" +# service.beta.kubernetes.io/oci-load-balancer-subnet1: "ocid1.subnet.oc1.ap-hyderabad-1.xxxxxxxxxxxx" #subnet id +# for OCI public +# nginx public - oci specific +#nginx_public_ingress_service_annotations: +# oci.oraclecloud.com/load-balancer-type: "lb" +# service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" +# service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "100Mbps" +# service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "200Mbps" +# service.beta.kubernetes.io/oci-load-balancer-subnet1: "ocid1.subnet.oc1.ap-hyderabad-1.xxxxxxxxxxx" #subnet id # for azure: # nginx_private_ingress_annotations: # service.beta.kubernetes.io/azure-load-balancer-internal: "true" @@ -113,10 +131,26 @@ cloud_storage_url: "https://{{ cloud_public_storage_accountname }}.blob.core.win nginx_private_ingress_annotations: service.beta.kubernetes.io/azure-load-balancer-internal: "true" +# Media specific vars - OCI ( enable when cloud_service_provider is oci) +#oci_media_region: "{{ cloud_public_storage_region }}" +#oci_media_compartment: ocid1.compartment.oc1..xxxxxxxxxxxx # compartment id +#oci_media_namespace: "{{ cloud_public_storage_namespace }}" +#oci_media_source_bucket: "{{ cloud_storage_content_bucketname }}" +#oci_media_target_bucket: "{{ cloud_storage_content_bucketname }}" +#oci_media_prefix_input: "" +#oci_media_dist_channel_id: "" +#oci_media_work_flow_id: "" +#oci_media_stream_config_id: "" +#oci_media_gateway_domain: "" +#csp_migrator_router_parallelism: "" # ------------------------------------------------------------------------------------------------------------ # graylog_trusted_proxies: "1.2.3.4/21" # This should be the kubernetes nodes subnet CIDR range +### Android Mobile deeplink url - Upload the assetlinks.json file to the public object store (bucket) +## reference: https://project-sunbird.atlassian.net/wiki/spaces/SBDES/pages/3276865567/Deep+link+support+in+Android+12+devices +mobile_deeplink_url: "{{ cloud_storage_url }}/{{ cloud_storage_public_bucketname }}/Mobile-deeplink/assetlinks.json" + ## Optional learningservice_ip: "10.0.1.5" # Load balancer IP or server ip @@ -484,12 +518,19 @@ portal_redis_connection_string: "redis://:@{{ sunbird_redis_host }}:6379/3" # U #### ML Analytics Service ##### ml_analytics_cname_url: # Reference value - https://obj.dev.sunbirded.org/samiksha/ +## ML internal kong #### +internal_kong_postgres_host: +internal_kong_ssl: True +internal_kong_postgres_user: +internal_kong_postgres_password: +internal_kong_postgres_database: + ### Variables added as part of 6.0.0 release - https://github.com/Sunbird-Ed/ml-analytics-service/tree/release-6.0.0/release-notes/6.0.0.md ml_analytics_client_id : project-sunbird-dev-client # Keycloak client group ml_analytics_username : reportAdmin1 # Username of a user which has roles of "PROGRAM_MANAGER", "PROGRAM_DESIGNER", "REPORT_ADMIN", "REPORT_VIEWER" ml_analytics_createdBy : fb85a044-d9eb-479b-a55a-faf1bfaea14d # Unique system generated user UUID which is the same user as above ml_container: samiksha - +ml_kafka_topic_create: "" # set to true if ml related topics needs to be created # provide the s3 compatible endpoint # for AWS # cloud_private_storage_account_endpoint: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" diff --git a/private_repo/ansible/inventory/dev/Core/secrets.yml b/private_repo/ansible/inventory/dev/Core/secrets.yml index c7f3e5b7e4..e511b72aac 100644 --- a/private_repo/ansible/inventory/dev/Core/secrets.yml +++ b/private_repo/ansible/inventory/dev/Core/secrets.yml @@ -5,7 +5,7 @@ core_vault_postgres_password: "change.postgres.password" # Update same core_vault_dp_pgdb_password: "change.postgres.password" #update same password in dp secrets dp_vault_pgdb_password variable core_vault_sunbird_azure_storage_key: "change.azure.storage.account.key" core_vault_docker_registry_url: "change.docker.url" # for docker hub "https://index.docker.io/v1" -core_vault_docker_registry_user: "change.docker.username" +core_vault_docker_registry_user: "change.docker.username" # For OCI oci_namespcae/oci_username core_vault_docker_registry_password: "change.docker.password" # Cloud Service Provider Secret Variables @@ -31,7 +31,7 @@ core_vault_docker_registry_password: "change.docker.password" # Example: cloud_management_storage_secret: "BfoOQwA6UFI7URwkFjkjdjsdhjhjhjsdhjfM//lsTbXDmHOFR5GqEwAwzyrKXOUDywojCpOC/g==" cloud_management_storage_secret: "" -# If cloud_service_provider is aws then update secret access key +# If cloud_service_provider is aws/oci then update secret access key # Example: cloud_public_storage_secret: "xkxjfjsdsfjdfjdhgjfhgjdhfgjdgjdfgdjgjdgdjhgjhd" # If cloud_service_provider is gcp then update with private-key which is in service account json file @@ -237,4 +237,8 @@ lp_vault_youtube_api_key: # youtube api token if you want # Examples values added below ml_analytics_authorization_access_token : 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJkOTY3NzRjYzXXXXXXXXXXXXXQ4Y2RiOWQ2Mzg0OSJ9.n4hXxKxl_698yeZPSWhXeGvMOb4esfgPadIZe8jZ0Z4' # Bearer auth token which has access to APIs as defined in the release note ml_analytics_password : Test@123 # Password of a user which has roles of "PROGRAM_MANAGER", "PROGRAM_DESIGNER", "REPORT_ADMIN", "REPORT_VIEWER" matching with supplied username to variable ml_analytics_username -ml_analytics_client_secret : fd241dce-4ZZ9-47e1-97cf-1c7de7a44216 # Keycloak client secret for creating tokens, mapped to client id supplied to variable ml_analytics_client_id \ No newline at end of file +ml_analytics_client_secret : fd241dce-4ZZ9-47e1-97cf-1c7de7a44216 # Keycloak client secret for creating tokens, mapped to client id supplied to variable ml_analytics_client_id + +#### ML Core Service ##### +ml_core_internal_access_token: 8c3a94f0934e01a4940a # THIS IS A REFERENCE VALUE ONLY.Secure token to access Admin-related APIs.The value of the internal access token can be generated by command - "openssl rand -hex 10" +ml_api_access_token: 8c3a94f0934e01a4940a # THIS IS A REFERENCE VALUE ONLY.Secure token to access Admin-related APIs.The value of the internal access token can be generated by command - "openssl rand -hex 10" diff --git a/private_repo/ansible/inventory/dev/DataPipeline/common.yml b/private_repo/ansible/inventory/dev/DataPipeline/common.yml index c5790ec42c..fa568ffdf2 100644 --- a/private_repo/ansible/inventory/dev/DataPipeline/common.yml +++ b/private_repo/ansible/inventory/dev/DataPipeline/common.yml @@ -6,25 +6,30 @@ domain_name: "" # your domain name like example.com dockerhub: "change.docker.url" # docker hub username or url incase of private registry private_ingressgateway_ip: "" # your private kubernetes load balancer ip search_lb_ip: # search service Load balancer IP +learningservice_ip: "10.0.1.5" # Load balancer IP or server ip # Cloud Service Provider Variables # If cloud_service_provider is AWS then update with access key as value # Example: cloud_public_storage_accountname: "AKSHKSJHFJDHJDSHFKSD" +# If cloud_service_provider is OCI then update with access key as value +# Example: cloud_public_storage_accountname: "7855461d26e1d8f211a04ec59cfd3037c8d1c6dd" # If cloud_service_provider is gloud(GCP) then update with service account name # Example: cloud_public_storage_accountname: "cloudstorage-gcp-test.iam.gserviceaccount.com" # If cloud_service_provider is AZURE then update with stoage account name # Example: cloud_public_storage_accountname: "azurestotageaccount" + cloud_public_storage_accountname: "" -# If cloud_service_provider is AWS then update with region +# If cloud_service_provider is AWS/OCI then update with region # Example: cloud_public_storage_region: us-east-1 cloud_public_storage_region: "" # If cloud_service_provider is gcp then update this variable with project id # Example: cloud_public_storage_project: "sunbird-gcp-test" cloud_public_storage_project: "" - - +# If cloud_service_provider is OCI then update this variable with namespace value +# Example: cloud_public_storage_namespace: "apsyjjclx" +cloud_public_storage_namespace: "" # Create object storage for each below mentioned variables and update accordingly -# If cloud_service_provider is AWS update with bucket name +# If cloud_service_provider is AWS/OCI update with bucket name # If cloud_service_provider is gcloud(GCP) update with bucket name # If cloud_service_provider is AZURE update with container name # Example: cloud_storage_certqr_bucketname: "certqr-storage" @@ -96,6 +101,8 @@ cloud_storage_management_bucketname: "" # cloud_storage_url: https://storage.googleapis.com # AWS # cloud_storage_url: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" +# OCI +# cloud_storage_url: "https://{{ cloud_public_storage_namespace }}.compat.objectstorage.{{ cloud_public_storage_region }}.oraclecloud.com" # Azure cloud_storage_url: "https://{{ cloud_public_storage_accountname }}.blob.core.windows.net" @@ -130,6 +137,11 @@ postgres: db_port: 5432 db_admin_user: postgres db_admin_password: "{{dp_vault_pgdb_admin_password}}" + dp_ssl: false + dp_ssl_mode: require + dp_sslfactory: org.postgresql.ssl.NonValidatingFactory + dp_db_name: analytics + spark_db_name: analytics druid_postgres_user: druid # Do not change this imagepullsecrets: "{{env}}registrysecret" # kubernetes imagePullSecrets @@ -178,6 +190,11 @@ cloudstorage_base_path: "{{ cloud_storage_url }}" valid_cloudstorage_base_urls: '["{{ cloud_storage_url }}"]' cloudstorage_relative_path_prefix: "CONTENT_STORAGE_BASE_PATH" +cloud_storage_telemetry_type: "{{ cloud_service_provider }}" +druid_storage_type: +spark_bds_url: +druid_cloud_storage_pathstyle_access: true +spark_version: # overide this var if you need a different version of spark. Ex: spark_version: 3.1.3 # The below sets the kafka topics retention time to 1 day, if you use the defaults from the public repo, it will be 7 days # If you want to retain the topics for 7 days, remove the below sections completely # Ensure you have atleast 1 TB of disk to retain data for 7 days @@ -295,4 +312,4 @@ processing_kafka_overriden_topics: replication_factor: 1 # graylog -send_logs_to_graylog: true # filebeat agents will send logs to graylog instead of ES \ No newline at end of file +send_logs_to_graylog: true # filebeat agents will send logs to graylog instead of ES diff --git a/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml b/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml index d87400b63c..5c13f9d9b9 100644 --- a/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml +++ b/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml @@ -5,16 +5,18 @@ cloud_service_provider: "" # Your cloud service provider name. Supported v dockerhub: "change.docker.url" # docker hub username or url incase of private registry private_ingressgateway_ip: "" # your private kubernetes load balancer ip domain_name: "" # your domain name like example.com +kp_print_service_base_url: "http://{{private_ingressgateway_ip}}/print" +learningservice_ip: "10.0.1.5" # Load balancer IP or server ip # Cloud Service Provider Variables -# If cloud_service_provider is AWS then update with access key as value +# If cloud_service_provider is AWS/OCI then update with access key as value # Example: cloud_public_storage_accountname: "AKSHKSJHFJDHJDSHFKSD" # If cloud_service_provider is gloud(GCP) then update with service account name # Example: cloud_public_storage_accountname: "cloudstorage-gcp-test.iam.gserviceaccount.com" # If cloud_service_provider is AZURE then update with stoage account name # Example: cloud_public_storage_accountname: "azurestotageaccount" cloud_public_storage_accountname: "" -# If cloud_service_provider is AWS then update with region +# If cloud_service_provider is AWS/OCI then update with region # Example: cloud_public_storage_region: us-east-1 cloud_public_storage_region: "" # If cloud_service_provider is gcp then update this variable with project id @@ -26,7 +28,7 @@ cloud_public_storage_namespace: "" # Create object storage for each below mentioned variables and update accordingly -# If cloud_service_provider is AWS update with bucket name +# If cloud_service_provider is AWS/OCI update with bucket name # If cloud_service_provider is gcloud(GCP) update with bucket name # If cloud_service_provider is AZURE update with container name # Example: cloud_storage_certqr_bucketname: "certqr-storage" @@ -98,6 +100,8 @@ cloud_storage_management_bucketname: "" # cloud_storage_url: https://storage.googleapis.com # AWS # cloud_storage_url: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" +# OCI +# cloud_storage_url: "https://{{ cloud_public_storage_namespace }}.compat.objectstorage.{{ cloud_public_storage_region }}.oraclecloud.com" # Azure cloud_storage_url: "https://{{ cloud_public_storage_accountname }}.blob.core.windows.net" @@ -129,8 +133,11 @@ kubeconfig_path: /var/lib/jenkins/secrets/k8s.yaml # kubeconfig file path # cloud_public_storage_endpoint: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" # for OCI #cloud_public_storage_endpoint: "https://.compat.objectstorage.{{cloud_public_storage_region}}.oraclecloud.com" - -cloud_public_storage_endpoint: "" +#oci_flink_s3_storage_endpoint: "{{ cloud_public_storage_endpoint }}" +#cloudstorage_sdk_endpoint: "{{ cloud_public_storage_endpoint }}" +#s3_region: "{{ cloud_public_storage_region }}" +#cloud_storage_proxy_host: "{{cloud_storage_url}}" +#flink_container_name: "{{ cloud_storage_flink_bucketname }}" # Update below vars if seperate object storage is required cloud_private_storage_accountname: "{{ cloud_public_storage_accountname }}" @@ -163,13 +170,26 @@ cloud_artifact_storage_namespace: "{{ cloud_public_storage_namespace }}" # cloud_storage_neo4jbackup_bucketname: "" # cloud_storage_redisbackup_bucketname: "" +# Media specific vars - OCI ( enable when cloud_service_provider is oci) +#oci_media_region: "{{ cloud_public_storage_region }}" +#oci_media_compartment: ocid1.compartment.oc1..xxxxxxxxxxxx # compartment id +#oci_media_namespace: "{{ cloud_public_storage_namespace }}" +#oci_media_source_bucket: "{{ cloud_storage_content_bucketname }}" +#oci_media_target_bucket: "{{ cloud_storage_content_bucketname }}" +#oci_media_prefix_input: "" +#oci_media_dist_channel_id: "" +#oci_media_work_flow_id: "" +#oci_media_stream_config_id: "" +#oci_media_gateway_domain: "" +#csp_migrator_router_parallelism: "" + # Building block vars cloud_storage_base_url: "{{ cloud_storage_url }}" cloudstorage_base_path: "{{ cloud_storage_url }}" valid_cloudstorage_base_urls: '["{{ cloud_storage_url }}"]' cloudstorage_relative_path_prefix: "CONTENT_STORAGE_BASE_PATH" -cloud_storage_pathstyle_access: false - +cloud_storage_pathstyle_access: true +cloud_storage_cname_url: "{{ cloud_storage_url }}" # overide if you have seperate url for cname ### Lern BB - Adding Lern specific vars here. In future if we want to move it to seperate folder this can be used as the starting point # Mandatorty variables - DO NOT LEAVE ANYTHING BLANK #