diff --git a/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js b/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js index 5b4b6c6125..1460374799 100644 --- a/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js +++ b/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js @@ -3159,6 +3159,7 @@ var handleGoogleAuthEvent = () => { redirectToLib(); } }; + var redirectToPortal = (redirectUrlPath) => { // redirectUrlPath for sso and self signUp const curUrlObj = window.location; var redirect_uri = getValueFromSession('redirect_uri'); @@ -3171,9 +3172,7 @@ var redirectToPortal = (redirectUrlPath) => { // redirectUrlPath for sso and sel const redirect_uriLocation = new URL(redirect_uri); if (client_id === 'android' || client_id === 'desktop') { window.location.href = sessionUrlObj.protocol + '//' + sessionUrlObj.host + redirectUrlPath + updatedQuery; - } else if(client_id === 'portal' && - redirectUrlPath === '/sign-in/sso/select-org' && - (redirect_uri.includes('dock.sunbirded.org') || redirect_uri.includes('dockstaging.sunbirded.org'))) { + } else if(client_id === 'portal' && redirectUrlPath === '/sign-in/sso/select-org' && (sessionUrlObj.host !== redirect_uriLocation.host)) { window.location.href = sessionUrlObj.protocol + '//' + sessionUrlObj.host + redirectUrlPath + updatedQuery; } else { window.location.href = redirect_uriLocation.protocol + '//' + redirect_uriLocation.host + diff --git a/ansible/roles/cassandra/defaults/main.yml b/ansible/roles/cassandra/defaults/main.yml index 33c02c17bf..8f1f0f7951 100644 --- a/ansible/roles/cassandra/defaults/main.yml +++ b/ansible/roles/cassandra/defaults/main.yml @@ -1,5 +1,5 @@ user_home: "/home/{{ ansible_ssh_user }}/" -cassandra_repo: 'deb http://www.apache.org/dist/cassandra/debian 311x main' -cassandra_repo_key: 'https://www.apache.org/dist/cassandra/KEYS' +cassandra_repo: 'deb https://debian.cassandra.apache.org 311x main' +cassandra_repo_key: 'https://downloads.apache.org/cassandra/KEYS' resource_crunch: "yes" # make this value yes if heap size should be quarter of the Server Memory when you are deploying multiple databases, else make this value no to have 50% of the value as per the best practice of cassandra cassandra_home_directory: "/var/lib/cassandra" diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 6090bfff0f..837c208319 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -8237,6 +8237,27 @@ kong_apis: config.required: false config.enabled: false +- name: uploadFileToCloud + uris: "{{ cloud_service_prefix }}/mlcore/v1/files/upload" + upstream_url: "{{ ml_core_service_url }}/v1/cloud-services/files/upload" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - cloudUrlsCreate + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + - name: listPlatformPrograms uris: "{{ user_extension_prefix }}/mlcore/v1/programsByPlatformRoles" upstream_url: "{{ ml_core_service_url }}/v1/user-extension/programsByPlatformRoles" diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index 15ef432e43..b85fe27059 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -133,3 +133,9 @@ ml_analytics_reports_store: "{{ cloud_service_provider }}" ml_analytics_reports_container: "{{ cloud_storage_privatereports_bucketname }}" ml_analytics_driver_memory: "{{ ml_analytics_default_driver_memory | default('5g') }}" ml_analytics_executor_memory: "{{ ml_analytics_default_executor_memory | default('5g') }}" +ml_analytics_batch_cron_minute: "{{ ml_batch_cron_minute | default('30') }}" +ml_analytics_batch_cron_hour: "{{ ml_batch_cron_hour | default('18') }}" +ml_analytics_nvsk_cron_minute: "{{ ml_nvsk_cron_minute | default('30') }}" +ml_analytics_nvsk_cron_hour: "{{ ml_nvsk_cron_hour | default('7') }}" +ml_analytics_nvsk_cron_weekday: "{{ ml_nvsk_cron_weekday | default('4') }}" +ml_analytics_faust_applications_list: "'observations/py_observation_streaming observations/', 'observations/py_observation_evidence_streaming observations/', 'survey/py_survey_streaming survey/', 'survey/py_survey_evidence_streaming survey/'" \ No newline at end of file diff --git a/ansible/roles/ml-analytics-service/tasks/main.yml b/ansible/roles/ml-analytics-service/tasks/main.yml index 6f37d6b3ea..7fbd3fe1ba 100755 --- a/ansible/roles/ml-analytics-service/tasks/main.yml +++ b/ansible/roles/ml-analytics-service/tasks/main.yml @@ -26,6 +26,7 @@ - python3-virtualenv - zip - unzip + - acl state: present - name: CHANGE THE OWNERSHIP FOR THIS {{ BASEPATH }} DIRECTORY @@ -135,15 +136,15 @@ cron: name: "Run Batch Ingestion Job" user: "{{ USER }}" - minute: "30" - hour: "18" + minute: "{{ ml_analytics_batch_cron_minute }}" + hour: "{{ ml_analytics_batch_cron_hour }}" job: "{{ BASEPATH }}/ml-analytics-service/run.sh > {{ BASEPATH }}/ml-analytics-service/crontab_job.log" - name: CREATE THE CRON NVSK Data Upload cron: name: "NVSK Data Upload JOB" user: "{{ USER }}" - minute: "30" - hour: "7" - weekday: "4" + minute: "{{ ml_analytics_nvsk_cron_minute }}" + hour: "{{ ml_analytics_nvsk_cron_hour }}" + weekday: "{{ ml_analytics_nvsk_cron_weekday }}" job: "{{ BASEPATH }}/ml-analytics-service/run_weekly.sh > {{ BASEPATH }}/ml-analytics-service/nvsk_data_weekly.logs" diff --git a/ansible/roles/ml-analytics-service/templates/config.j2 b/ansible/roles/ml-analytics-service/templates/config.j2 index 935085d36a..201e9ab8b5 100644 --- a/ansible/roles/ml-analytics-service/templates/config.j2 +++ b/ansible/roles/ml-analytics-service/templates/config.j2 @@ -263,3 +263,8 @@ store = {{ ml_analytics_reports_store }} [ML_CORE_SERVICE_URL] url = {{ ml_analytics_core_service }} + + +[SERVICES] + +faust_applications_list = {{ ml_analytics_faust_applications_list }} diff --git a/ansible/roles/stack-sunbird/defaults/main.yml b/ansible/roles/stack-sunbird/defaults/main.yml index 581f28458b..0c0a678be5 100644 --- a/ansible/roles/stack-sunbird/defaults/main.yml +++ b/ansible/roles/stack-sunbird/defaults/main.yml @@ -268,6 +268,7 @@ service_env: ml-reports-service: ../../../../ansible/roles/stack-sunbird/templates/ml-reports-service.env ml-core-service: ../../../../ansible/roles/stack-sunbird/templates/ml-core-service.env ml-projects-service: ../../../../ansible/roles/stack-sunbird/templates/ml-projects-service.env + ml-analytics-service: ../../../../ansible/roles/stack-sunbird/templates/ml-analytics-service.conf inbound: ../../../../ansible/roles/stack-sunbird/templates/inbound.env outbound: - ../../../../ansible/roles/stack-sunbird/templates/outbound.env diff --git a/ansible/roles/stack-sunbird/templates/analytics_api_service.conf b/ansible/roles/stack-sunbird/templates/analytics_api_service.conf index 8bdcd63467..b9a8c838c8 100644 --- a/ansible/roles/stack-sunbird/templates/analytics_api_service.conf +++ b/ansible/roles/stack-sunbird/templates/analytics_api_service.conf @@ -201,10 +201,10 @@ public { } } cloud_storage_type="{{cloud_service_provider}}" -storage.key.config="{{cloud_public_storage_accountname}}" -storage.secret.config="{{cloud_public_storage_secret}}" -public.storage.key.config="{{cloud_public_storage_accountname}}" -public.storage.secret.config="{{cloud_public_storage_secret}}" +storage.key.config="cloud_storage_key" +storage.secret.config="cloud_storage_secret" +public.storage.key.config="public_cloud_storage_key" +public.storage.secret.config="public_cloud_storage_secret" cloud_storage_endpoint_with_protocol="{{cloud_storage_url}}" metrics.time.interval.min=30 cache.refresh.time.interval.min=1440 diff --git a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf new file mode 100644 index 0000000000..707dc83e6f --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf @@ -0,0 +1,294 @@ +[ML_APP_NAME] + +survey_app = {{ ml_survey_app_name | default('sunbirdsurvey') }} + +integrated_app = {{ ml_integrated_app_name | default('sunbird') }} + +integrated_portal = {{ ml_integrated_portal | default('dev.sunbird.portal') }} + + +[API_ENDPOINTS] + +base_url = https://{{ domain_name }}/ + + +[API_HEADERS] + +content_type = application/json + +authorization = {{ml_api_auth_token | default('sunbird_api_auth_token')}} + +internal_access_token = {{ml_api_access_token | default('ml_core_internal_access_token')}} + +authorization_access_token = {{ ml_analytics_authorization_access_token }} + +[ML_SURVEY_SERVICE_URL] + +url = http://{{private_ingressgateway_ip}}/ml-survey + +entity_related_end_point = assessment/api/v1/entities/relatedEntities/ + +user_profile_end_point = assessment/api/v1/userExtension/getProfile/ + +evidence_base_url = {{ cloud_storage_url }}/{{ ml_analytics_public_storage }}/ + + +[MONGO] + +url = mongodb://{{ml_mongodb_host | default(groups['mongo_master'][0]+':27017')}} + +database_name = {{ml_mongodb | default('ml-survey')}} + + +# ------ Mongo Collections ------- # +observation_sub_collection = observationSubmissions + +solutions_collection = solutions + +observations_collection = observations + +entity_type_collection = entityTypes + +questions_collection = questions + +criteria_collection = criteria + +entities_collection = entities + +programs_collection = programs + +user_roles_collection = userRoles + +criteria_questions_collection = criteriaQuestions + +projects_collection = projects + +survey_submissions_collection = surveySubmissions + +survey_collection = surveys + +[DRUID] + +metadata_url = http://{{groups['druid'][0]}}:8081/druid/coordinator/v1/datasources/ + +batch_url = http://{{groups['druid'][0]}}:8081/druid/indexer/v1/task + +metadata_rollup_url = http://{{groups['druid'][0]}}:8081/druid/coordinator/v1/datasources/ + +batch_rollup_url = http://{{groups['druid'][0]}}:8081/druid/indexer/v1/task + +urlQuery = http://{{groups['druid'][0]}}:8082/druid/v2?pretty + +datasource_url = http://{{groups['ml-raw-broker'][0]}}:{{ml_raw_broker_port}}/druid/v2/datasources + +sql_url = http://{{groups['ml-raw-broker'][0]}}:{{ml_raw_broker_port}}/druid/v2/sql/ + +coordinator_url = http://{{groups['ml-raw-coordinator'][0]}}:{{ml_raw_coordinator_port}}/status/health + +overload_url = http://{{groups['ml-raw-overload'][0]}}:{{ml_raw_overload_port}}/status/health + +historical_url = http://{{groups['ml-raw-historical'][0]}}:{{ml_raw_historical_port}}/status/health + +observation_status_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_code","school_externalId","state_name","state_code","state_externalId","district_name","district_code","district_externalId","block_name","block_code","block_externalId","cluster_name","cluster_code","cluster_externalId","completedDate","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name","ecm_marked_na","board_name","updatedAt","organisation_id","user_type","observed_school_name","observed_school_id","observed_school_code","observed_state_name","observed_state_id","observed_state_code","observed_district_name","observed_district_id","observed_district_code","observed_block_name","observed_block_id","observed_block_code","observed_cluster_name","observed_cluster_id","observed_cluster_code","isRubricDriven","criteriaLevelReport"]},"metricsSpec":[]}}}' + +project_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-project","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"auto"},"dimensionsSpec":{"dimensions":[]},"metricsSpec":[]}}}' + +ml_distinctCnt_obs_status_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' + +ml_distinctCnt_obs_domain_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' + +ml_distinctCnt_obs_domain_criteria_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain-criteria","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"string","name":"criteria_name"},{"type":"string","name":"criteria_score"},{"type":"string","name":"criteria_id"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' + +ml_distinctCnt_projects_status_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"project_title"},{"type":"string","name":"solution_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"long","name":"no_of_imp_with_evidence_inprogress"},{"type":"string","name":"time_stamp"},{"type":"long","name":"no_of_certificate_issued"}]},"metricsSpec":[]}}}' + +ml_distinctCnt_prglevel_projects_status_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-programLevel-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"},{"type":"long","name":"no_of_certificate_issued"}]},"metricsSpec":[]}}}' + +ml_distinctCnt_survey_status_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-surveydistinctCount-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"survey_name"},{"type":"string","name":"survey_id"},{"type":"string","name":"submission_status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_submissions"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' + +observation_status_rollup_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-observation-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["status","user_id","solution_id","submission_id","entity_name","completedDate","program_id","private_program","solution_type","updatedAt","role_title","solution_name","program_name","channel","parent_channel","block_name","district_name","school_name","cluster_name","state_name","organisation_name","board_name","district_externalId","state_externalId","block_externalId","cluster_externalId","school_externalId","organisation_id",{"type":"long","name":"status_code"}]},"metricsSpec":[{"type":"count","name":"count"},{"type":"longSum","name":"sum___v","fieldName":"__v","expression":null},{"type":"HLLSketchBuild","name":"count_distinct_solution","fieldName":"solution_id","lgK":12,"tgtHllType":"HLL_4","round":false},{"type":"HLLSketchBuild","name":"count_distinct_submission_id","fieldName":"submission_id","lgK":12,"tgtHllType":"HLL_4","round":false},{"type":"HLLSketchBuild","name":"count_distinct_user_id","fieldName":"user_id","lgK":12,"tgtHllType":"HLL_4","round":false}]}}}' + +project_rollup_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"iso"},"dimensionsSpec":{"dimensions":["project_title","project_goal","area_of_improvement","status_of_project","tasks_name","tasks_status","designation","task_evidence_status","project_id","task_id","project_created_type","parent_channel","program_id","program_name","project_updated_date","createdBy","program_externalId","private_program","task_deleted_flag","project_terms_and_condition","state_externalId","block_externalId","district_externalId","cluster_externalId","school_externalId","state_name","block_name","district_name","cluster_name","school_name","board_name","organisation_name","solution_id","organisation_id",{"name":"status_code","type":"long"}]},"metricsSpec":[{"name":"count","type":"count"},{"name":"sum___v","type":"longSum","fieldName":"__v"},{"name":"sum_status_code","type":"longMax","fieldName":"status_code"},{"type":"HLLSketchBuild","name":"count_of_createBy","fieldName":"createdBy"},{"type":"HLLSketchBuild","name":"count_of_project_id","fieldName":"project_id"},{"type":"HLLSketchBuild","name":"count_of_solution_id","fieldName":"solution_id"},{"type":"HLLSketchBuild","name":"count_of_program_id","fieldName":"program_id"}]}}}' + +ml_survey_rollup_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel",{"type":"long","name":"status_code"}, "solution_name", "solution_id"]},"metricsSpec":[{"name":"count","type":"count"},{"name":"sum___v","type":"longSum","fieldName":"__v"},{"name":"sum_status_code","type":"longMax","fieldName":"status_code"},{"type":"HLLSketchBuild","name":"count_of_user_id","fieldName":"user_id"},{"type":"HLLSketchBuild","name":"count_of_survey_submission_id","fieldName":"survey_submission_id"},{"type":"HLLSketchBuild","name":"count_of_solution_id","fieldName":"solution_id"},{"type":"HLLSketchBuild","name":"count_of_program_id","fieldName":"program_id"}]}}}' + +survey_status_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel", "solution_name", "solution_id","private_program","state_code","school_code","district_code","block_code","cluster_code"]},"metricsSpec":[]}}}' + +observation_query_spec = '{"queryType":"scan","dataSource":"sl-observation","resultFormat":"list","columns":["completedDate","createdAt","createdBy","criteriaExternalId","criteriaId","criteriaName","entityType","entityTypeId","observationId","observationName","observationSubmissionId","questionAnswer","questionECM","questionExternalId","questionId","questionName","questionResponseLabel","questionResponseType","solutionExternalId","solutionId","solutionName","updatedAt","instanceParentId","instanceId","instanceParentResponsetype","instanceParentQuestion","questionSequenceByEcm","maxScore","minScore","percentageScore","pointsBasedScoreInParent","totalScore","scoreAchieved","totalpercentage","instanceParentExternalId","instanceParentEcmSequence","remarks","total_evidences","evidence_count","school","block","district","cluster","state","schoolName","blockName","districtName","clusterName","stateName","schoolExternalId","blockExternalId","districtExternalId","clusterExternalId","stateExternalId","schoolTypes","administrationTypes","instanceParentCriteriaId","instanceParentCriteriaExternalId","instanceParentCriteriaName","role_title","location_validated_with_geotag","distance_in_meters","entity","entityExternalId","entityName","isAPrivateProgram","programId","programName","programExternalId","questionResponseLabel_number","criteriaLevel","criteriaScore","submissionNumber","submissionTitle","channel","parent_channel","user_districtName","user_blockName","user_clusterName","appName","evidences","user_stateName","domainName","domainExternalId","childName","childType","childExternalid","level","criteriaDescription","programDescription","solutionDescription","label","imp_project_id","imp_project_title","imp_project_goal","imp_project_externalId","ancestorName","scoringSystem","domainLevel","domainScore","criteriaLevelReport","user_schoolName","user_schoolId","user_schoolUDISE_code","solution_type","organisation_name","user_boardName","district_externalId","state_externalId","block_externalId","cluster_externalId","organisation_id","user_type"],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]}' + +observation_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"entityType"},{"type":"string","name":"entityTypeId"},{"type":"string","name":"observationId"},{"type":"string","name":"observationName"},{"type":"string","name":"observationSubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"school"},{"type":"string","name":"block"},{"type":"string","name":"district"},{"type":"string","name":"cluster"},{"type":"string","name":"state"},{"type":"string","name":"schoolName"},{"type":"string","name":"blockName"},{"type":"string","name":"districtName"},{"type":"string","name":"clusterName"},{"type":"string","name":"stateName"},{"type":"string","name":"schoolExternalId"},{"type":"string","name":"blockExternalId"},{"type":"string","name":"districtExternalId"},{"type":"string","name":"clusterExternalId"},{"type":"string","name":"stateExternalId"},{"type":"string","name":"schoolTypes"},{"type":"string","name":"administrationTypes"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"role_title"},{"type":"string","name":"location_validated_with_geotag"},{"type":"string","name":"distance_in_meters"},{"type":"string","name":"entity"},{"type":"string","name":"entityExternalId"},{"type":"string","name":"entityName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"criteriaLevel"},{"type":"string","name":"criteriaScore"},{"type":"string","name":"submissionNumber"},{"type":"string","name":"submissionTitle"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"user_districtName"},{"type":"string","name":"user_blockName"},{"type":"string","name":"user_clusterName"},{"type":"string","name":"appName"},{"type":"string","name":"evidences"},{"type":"string","name":"user_stateName"},{"type":"string","name":"domainName"},{"type":"string","name":"domainExternalId"},{"type":"string","name":"childName"},{"type":"string","name":"childType"},{"type":"string","name":"childExternalid"},{"type":"string","name":"level"},{"type":"string","name":"criteriaDescription"},{"type":"string","name":"programDescription"},{"type":"string","name":"solutionDescription"},{"type":"string","name":"label"},{"type":"string","name":"imp_project_id"},{"type":"string","name":"imp_project_title"},{"type":"string","name":"imp_project_goal"},{"type":"string","name":"imp_project_externalId"},{"type":"string","name":"ancestorName"},{"type":"string","name":"scoringSystem"},{"type":"string","name":"domainLevel"},{"type":"string","name":"domainScore"},{"name":"criteriaLevelReport","type":"boolean"},{"type":"string","name":"user_schoolName"},{"type":"string","name":"user_schoolId"},{"type":"string","name":"user_schoolUDISE_code"},{"type":"string","name":"solution_type"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_boardName"},{"type":"string","name":"district_externalId"},{"type":"string","name":"state_externalId"},{"type":"string","name":"block_externalId"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"user_type"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}' + +survey_query_spec = '{"queryType":"scan","dataSource":"sl-survey","resultFormat":"list","columns":["completedDate","createdAt","createdBy","criteriaExternalId","criteriaId","criteriaName","surveyId","surveyName","surveySubmissionId","questionAnswer","questionECM","questionExternalId","questionId","questionName","questionResponseLabel","questionResponseType","solutionExternalId","solutionId","solutionName","updatedAt","instanceParentId","instanceId","instanceParentResponsetype","instanceParentQuestion","questionSequenceByEcm","maxScore","minScore","percentageScore","pointsBasedScoreInParent","totalScore","scoreAchieved","totalpercentage","instanceParentExternalId","instanceParentEcmSequence","remarks","total_evidences","evidence_count","instanceParentCriteriaId","instanceParentCriteriaExternalId","instanceParentCriteriaName","isAPrivateProgram","programId","programName","programExternalId","questionResponseLabel_number","channel","parent_channel","appName","organisation_name","user_subtype","user_type","board_name","district_code","district_name","district_externalId","block_code","block_name","block_externalId","school_code","school_name","school_externalId","cluster_code","cluster_name","cluster_externalId","state_code","state_name","state_externalId","organisation_id","evidences"],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]}' + +survey_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-survey","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"surveyId"},{"type":"string","name":"surveyName"},{"type":"string","name":"surveySubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"evidences"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"appName"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_subtype"},{"type":"string","name":"user_type"},{"type":"string","name":"board_name"},{"type":"string","name":"district_code"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_code"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"school_code"},{"type":"string","name":"school_name"},{"type":"string","name":"school_externalId"},{"type":"string","name":"cluster_code"},{"type":"string","name":"cluster_name"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"state_code"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}' + +intervals = '["1901-01-01T00:00:00+00:00/2020-01-01T00:00:00+00:00","2020-01-01T00:00:00+00:00/2020-06-01T00:00:00+00:00","2020-06-01T00:00:00+00:00/2021-01-01T00:00:00+00:00","2021-01-01T00:00:00+00:00/2021-06-01T00:00:00+00:00","2021-06-01T00:00:00+00:00/2022-01-01T00:00:00+00:00","2022-01-01T00:00:00+00:00/2022-03-01T00:00:00+00:00","2022-03-01T00:00:00+00:00/2022-06-01T00:00:00+00:00","2022-06-01T00:00:00+00:00/2022-09-01T00:00:00+00:00","2022-09-01T00:00:00+00:00/2023-01-01T00:00:00+00:00"]' + + +[KAFKA] + +url = {{ml_kafka_host | default(groups['processing-cluster-kafka'][0]+':9092')}} + +observation_raw_topic = {{ env_name }}.ml.observation.raw + +observation_druid_topic = {{ env_name }}.ml.observation.druid + +observation_evidence_druid_topic = {{ env_name }}.ml.observation.evidence.druid + +survey_evidence_druid_topic = {{ env_name }}.ml.survey.evidence.druid + +survey_raw_topic = {{ env_name }}.ml.survey.raw + +survey_druid_topic = {{ env_name }}.ml.survey.druid + +survey_meta_druid_topic = {{ env_name }}.ml.survey.meta + +survey_completed_druid_topic = {{ env_name }}.ml.survey.status.completed + +survey_started_druid_topic = {{ env_name }}.ml.survey.status.started + +survey_inprogress_druid_topic = {{ env_name }}.ml.survey.status.inprogress + +observation_meta_druid_topic = {{ env_name }}.ml.observation.meta + +observation_started_druid_topic = {{ env_name }}.ml.observation.status.started + +observation_inprogress_druid_topic = {{ env_name }}.ml.observation.status.inprogress + +observation_completed_druid_topic = {{ env_name }}.ml.observation.status.completed + +[LOGS] + +observation_streaming_success_error = /opt/sparkjobs/ml-analytics-service/logs/observation/ + +observation_streaming_evidence_success_error = /opt/sparkjobs/ml-analytics-service/logs/observation/evidence/ + +observation_status_success_error = /opt/sparkjobs/ml-analytics-service/logs/observation/status/ + +project_success_error = /opt/sparkjobs/ml-analytics-service/logs/project/ + +survey_evidence_streaming_success_error = /opt/sparkjobs/ml-analytics-service/logs/project/evidence/ + +survey_streaming_success_error = /opt/sparkjobs/ml-analytics-service/logs/survey/ + +survey_status_success_error = /opt/sparkjobs/ml-analytics-service/logs/survey/status/ + + +[CLOUD] + +container_name = {{ cloud_storage_telemetry_bucketname }} + +[OUTPUT_DIR] + +project = /opt/sparkjobs/source/projects/output + +observation_status = /opt/sparkjobs/source/observations/status/output + +observation_distinctCount_status = /opt/sparkjobs/source/observations/distinctCount/output + +observation_distinctCount_domain = /opt/sparkjobs/source/observations/distinctCount_domain/output + +observation_distinctCount_domain_criteria = /opt/sparkjobs/source/observations/distinctCount_domain_criteria/output + +projects_distinctCount = /opt/sparkjobs/source/projects/distinctCount/output + +projects_distinctCount_prgmlevel = /opt/sparkjobs/source/projects/distinctCountPrglevel/output + +survey_distinctCount_status = /opt/sparkjobs/source/survey/distinctCount/output + +project_rollup = /opt/sparkjobs/source/projects/output_rollup + +observation_status_rollup = /opt/sparkjobs/source/observations/output_rollup + +survey_rollup = /opt/sparkjobs/source/survey/output + +survey_status = /opt/sparkjobs/source/survey/status/output + +observation_sub_ids = /opt/sparkjobs/ml-analytics-service/observations/submissions.csv + +observation_druid_data = /opt/sparkjobs/source/observations/ + +survey_sub_ids = /opt/sparkjobs/ml-analytics-service/survey/submissions.csv + +survey_druid_data = /opt/sparkjobs/source/survey/ + +program_text_file = /opt/sparkjobs/ml-analytics-service/projects/program_ids.txt + +[SLACK] + +token = {{ ml_analytics_slack_token | default('') }} + +channel = {{ ml_analytics_slack_channel | default('') }} + + +[VAM] + +druid_query_url = {{ ml_druid_query | default('') }} + +program_dashboard_data = {{ ml_program_data | default('') }} + + +[COMMON] + +cloud_module_path = /opt/sparkjobs/ml-analytics-service/cloud_storage + +observation_blob_path = observation/status/ + +projects_blob_path = projects/ + +observation_distinctCount_blob_path = observation/distinctCount/ + +observation_distinctCount_domain_blob_path = observation/distinctCount_domain/ + +observation_distinctCount_domain_criteria_blob_path = observation/distinctCount_domain_criteria/ + +projects_distinctCnt_blob_path = projects/distinctCount/ + +projects_distinctCnt_prgmlevel_blob_path = projects/distinctCountPrglevel/ + +survey_distinctCount_blob_path = survey/distinctCount/ + +projects_rollup_blob_path = projects/rollup + +observation_rollup_blob_path = observation/rollup + +survey_rollup_blob_path = survey/rollup/ + +survey_blob_path = survey/status/ + +projects_program_csv = {{ ml_program_blob_path | default('') }} + +observation_batch_ingestion_data_del = observation/batchDeletion + +survey_batch_ingestion_data_del = survey/batchDeletion + +cname_url = {{ cloud_storage_url }}/{{ cloud_storage_samiksha_bucketname }} + +nvsk_imp_projects_data_local_path = /opt/sparkjobs/ml-analytics-service/urgent_data_metrics/output/ + +nvsk_imp_projects_data_blob_path = Manage_Learn_Data/micro_improvement/ + +[API_CREDENTIALS] + +client_id = {{ ml_analytics_client_id }} + +client_secret = {{ ml_analytics_client_secret }} + +username = {{ ml_analytics_username }} + +password = {{ ml_analytics_password }} + +[JSON_VARIABLE] + +createdBy = {{ ml_analytics_createdBy }} + +container = {{ ml_analytics_reports_container }} + +store = {{ ml_analytics_reports_store }} + +[ML_CORE_SERVICE_URL] + +url = http://{{private_ingressgateway_ip}}/ml-core + +[SERVICES] + +faust_applications_list = {{ faust_applications_list }} diff --git a/ansible/roles/stack-sunbird/templates/ml-core-service.env b/ansible/roles/stack-sunbird/templates/ml-core-service.env index 56313ff44f..463c010d1b 100755 --- a/ansible/roles/stack-sunbird/templates/ml-core-service.env +++ b/ansible/roles/stack-sunbird/templates/ml-core-service.env @@ -69,4 +69,12 @@ KAFKA_GROUP_ID={{ml_core_kafka_group_id | default("ml-core-service")}} PROGRAM_USERS_JOINED_TOPIC={{ml_core_program_users_joined_topic | default (env_name+".programuser.info")}} #TimeOffset with respect to UTC -TIMEZONE_DIFFRENECE_BETWEEN_LOCAL_TIME_AND_UTC={{ml_core_timezone_diffrenece_between_local_time_and_utc | default ("+05:30")}} \ No newline at end of file +TIMEZONE_DIFFRENECE_BETWEEN_LOCAL_TIME_AND_UTC={{ml_core_timezone_diffrenece_between_local_time_and_utc | default ("+05:30")}} + + +#Validate Entities enable/Disable +VALIDATE_ENTITIES={{ml_service_validate_entities | default ("ON")}} + +#Public base url of sunbird +PUBLIC_BASE_URL={{ml_core_public_base_url | default("")}} + diff --git a/ansible/roles/stack-sunbird/templates/ml-survey-service.env b/ansible/roles/stack-sunbird/templates/ml-survey-service.env index ccda45606e..51f5828455 100644 --- a/ansible/roles/stack-sunbird/templates/ml-survey-service.env +++ b/ansible/roles/stack-sunbird/templates/ml-survey-service.env @@ -65,3 +65,9 @@ KEYCLOAK_PUBLIC_KEY_PATH={{ml_survey_keycloak_public_key_path|default("keycloak- ## portal url of env APP_PORTAL_BASE_URL={{ proto }}://{{ domain_name }} + +#Program join enable/disable +PROGRAM_JOIN_ON_OFF={{ml_program_join_on_off | default ("ON")}} + +#validate entities enable/disable +VALIDATE_ENTITIES={{ml_service_validate_entities | default ("ON")}} diff --git a/ansible/roles/stack-sunbird/templates/sunbird_report-service.env b/ansible/roles/stack-sunbird/templates/sunbird_report-service.env index a2ebfce02a..b13358fcac 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_report-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_report-service.env @@ -29,3 +29,13 @@ sunbird_cache_ttl={{sunbird_cache_ttl}} sunbird_report_sas_expiry_in_minutes=3600 sunbird_dataservice_url={{sunbird_dataservice_url}} +# 6.0.0 + +sunbird_cloud_storage_provider={{cloud_service_provider}} +cloud_private_storage_accountname={{cloud_private_storage_accountname | default("")}} +cloud_private_storage_secret={{cloud_private_storage_secret | default("")}} +cloud_storage_privatereports_bucketname={{cloud_storage_privatereports_bucketname | default("reports")}} +cloud_storage_resourceBundle_bucketname={{cloud_storage_label_bucketname | default("label")}} +cloud_private_storage_region={{cloud_private_storage_region | default("ap-south-1")}} +cloud_private_storage_project={{cloud_private_storage_project | default("")}} +cloud_private_storage_endpoint={{cloud_private_storage_endpoint | default("")}} diff --git a/deploy/jenkins/jenkins-server-setup.sh b/deploy/jenkins/jenkins-server-setup.sh index d96e3b4228..2c7516a57a 100755 --- a/deploy/jenkins/jenkins-server-setup.sh +++ b/deploy/jenkins/jenkins-server-setup.sh @@ -12,10 +12,10 @@ echo -e "\n\e[0;32m${bold}Installating JDK8${normal}\n" apt-get install -y openjdk-8-jdk echo -e "\n\e[0;32m${bold}Installating Jenkins${normal}" -wget -q -O - https://pkg.jenkins.io/debian-stable/jenkins.io.key | apt-key add - +wget -q -O - https://pkg.jenkins.io/debian-stable/jenkins.io-2023.key | apt-key add - apt-add-repository "deb https://pkg.jenkins.io/debian-stable binary/" apt-get update -apt-get install -y jenkins=2.319.3 +apt-get install -y jenkins=2.346.3 echo -e "\n\e[0;32m${bold}Installating PIP${normal}" apt-get install -y python-pip @@ -115,6 +115,7 @@ su jenkins bash -c "curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34. echo -e "\n\e[0;32m${bold}Installing jmespath${normal}" sudo apt install -y python3-jmespath +sudo apt install python-jmespath #python2 #### Kubernetes Tools #### @@ -155,7 +156,7 @@ rm openjdk-11.0.2_linux-x64_bin.tar.gz #Install maven 3.6.3 echo -e "\n\e[0;32m${bold}Installating maven 3.6.3${normal}" -wget https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz +wget https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz tar -xf apache-maven-3.6.3-bin.tar.gz mv apache-maven-3.6.3 /opt/ mv /opt/apache-maven-3.6.3/bin/mvn /opt/apache-maven-3.6.3/bin/mvn3.6 diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml new file mode 100644 index 0000000000..28b220559d --- /dev/null +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml @@ -0,0 +1,108 @@ + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + 10 + -1 + 1 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + Build/KnowledgePlatform/InquiryFlinkJob + false + + + image_tag + <font color=darkgreen size=2><b>OPTIONAL: Specify the tag to upload a specific image version to the container registry.</b></font> + + false + + + artifact_source + <font color=dimgray size=2><b> +ArtifactRepo - Push the docker image to container registry. +</b></font> + + + ArtifactRepo + + + + + + + 0 + 0 + + false + project + false + + + + + + + + Build/KnowledgePlatform/InquiryFlinkJob + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/Sunbird-inQuiry/data-pipeline.git + + + + + ${inquiry_pipeline_branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + kubernetes/pipelines/upload/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml index 5bc1fe4d34..c6304ef3b4 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml @@ -35,6 +35,30 @@ refs/heads/${public_repo_branch} true + + CLOUD_STORE_GROUP_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk group id</li> +</b></font> + org.sunbird + false + + + CLOUD_STORE_ARTIFACT_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk artifact id</li> +</b></font> + cloud-store-sdk_2.12 + false + + + CLOUD_STORE_VERSION + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk version</li> +</b></font> + 1.4.0 + false + @@ -48,12 +72,7 @@ - - - H/15 * * * * - false - - + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml index d2c038b5fa..b4deb5e46b 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml @@ -1,6 +1,6 @@ - + hudson.model.ParametersDefinitionProperty com.sonyericsson.rebuild.RebuildSettings @@ -19,26 +19,47 @@ - + false false - github_release_tag + inquiry_release_tag <font style="color:dimgray;font-size:14px;"><b> <li>To build from a tag, use refs/tags/github_tag</li> <li>To build from a branch, use refs/heads/github_branch</li> -<li>The default value of ${public_repo_branch} will be the release / tag version set in global configuration</li> -<li>To build from a differnt branch, replace the ${public_repo_branch} with your branch</li> +<li>The default value of ${inquiry_service_build_branch_or_tag} will be the release / tag version set in global configuration</li> +<li>To build from a differnt branch, replace the ${inquiry_service_build_branch_or_tag} with your branch</li> </b></font> - refs/heads/${public_repo_branch} + refs/heads/${inquiry_service_build_branch_or_tag} true + + core_release_tag + <font style="color:dimgray;font-size:14px;"><b> +<li>Provide this value to build core components from Knowlg BB</li> +<li>To build from a tag, use refs/tags/github_tag</li> +<li>To build from a branch, use refs/heads/github_branch</li> +<li>The default value of ${inquiry_core_build_branch_or_tag} will be the release / tag version set in global configuration</li> +<li>To build from a differnt branch, replace the ${inquiry_core_build_branch_or_tag} with your branch</li> +</b></font> + refs/heads/${inquiry_core_build_branch_or_tag} + false + + + core_repo_link + Knowlg Core Repo + + + https://github.com/project-sunbird/knowledge-platform.git + + + - + 0 0 @@ -49,31 +70,29 @@ - - - H/15 * * * * - false - - + - - + + 2 - https://github.com/project-sunbird/knowledge-platform.git + https://github.com/Sunbird-inQuiry/inquiry-api-service.git - ${github_release_tag} + ${inquiry_release_tag} + false + + build/assessment-service/Jenkinsfile false false - \ No newline at end of file + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml index 7b47e1972e..f3f5d6477c 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + CLOUD_STORE_GROUP_ID + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + CLOUD_STORE_ARTIFACT_ID + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12 + ${cloud_store_artifact_id} + false + + + CLOUD_STORE_VERSION + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml index 2cdac260ad..a12dd235f1 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + cloud_store_group_id + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + cloud_store_artifact_id + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk + ${cloud_store_artifact_id} + false + + + cloud_store_version + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + @@ -51,7 +69,7 @@ - H/15 * * * * + false diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml index 89116c8406..4eecb95a61 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + cloud_store_group_id + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + cloud_store_artifact_id + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk + ${cloud_store_artifact_id} + false + + + cloud_store_version + Set the Cloud store sdk version. e.g 1.4.6 + ${cloud_store_version} + false + diff --git a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/AnalyticsCore/config.xml b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/AnalyticsCore/config.xml index 2b9e72c45c..5c202a759c 100644 --- a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/AnalyticsCore/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/AnalyticsCore/config.xml @@ -36,6 +36,30 @@ refs/heads/${public_repo_branch} true + + CLOUD_STORE_GROUP_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk group id</li> +</b></font> + org.sunbird + false + + + CLOUD_STORE_ARTIFACT_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk artifact id</li> +</b></font> + cloud-store-sdk_2.12 + false + + + CLOUD_STORE_VERSION + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk version</li> +</b></font> + 1.4.0 + false + @@ -51,7 +75,7 @@ - H/15 * * * * + false diff --git a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/CoreDataProducts/config.xml b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/CoreDataProducts/config.xml index a458418b24..8ff4dc5426 100644 --- a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/CoreDataProducts/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/CoreDataProducts/config.xml @@ -27,6 +27,27 @@ github_release_tag + <font color=red size=2><b>CAUTION: If the value is blank, latest code will be built. Specify github tag name to build from a tag.</b></font> + false + + + CLOUD_STORE_GROUP_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk group id</li> +</b></font> + org.sunbird + false + + + CLOUD_STORE_ARTIFACT_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk artifact id</li> +</b></font> + cloud-store-sdk_2.12 + false + + + CLOUD_STORE_VERSION <font style="color:dimgray;font-size:14px;"><b> <li>To build from a tag, use refs/tags/github_tag</li> <li>To build from a branch, use refs/heads/github_branch</li> @@ -51,7 +72,7 @@ - H/15 * * * * + false diff --git a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml index 11f30a0d33..62a0cd1f68 100644 --- a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + CLOUD_STORE_GROUP_ID + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + CLOUD_STORE_ARTIFACT_ID + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12 + ${cloud_store_artifact_id} + false + + + CLOUD_STORE_VERSION + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + diff --git a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml new file mode 100644 index 0000000000..93b0eba8f7 --- /dev/null +++ b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml @@ -0,0 +1,77 @@ + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + 10 + -1 + 1 + + + + + false + false + + + + + inquiry_release_tag + <font style="color:dimgray;font-size:14px;"><b> +<li>To build from a tag, use refs/tags/github_tag</li> +<li>To build from a branch, use refs/heads/github_branch</li> +<li>The default value of ${inquiry_pipeline_branch_or_tag} will be the release / tag version set in global configuration</li> +<li>To build from a differnt branch, replace the ${inquiry_pipeline_branch_or_tag} with your branch</li> +</b></font> + refs/heads/${inquiry_pipeline_branch_or_tag} + true + + + + + 0 + 0 + + false + project + false + + + + + + + + + + 2 + + + https://github.com/Sunbird-inQuiry/data-pipeline.git + + + + + ${inquiry_release_tag} + + + false + + + + kubernetes/pipelines/build/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml index 088b9fa27f..9b997d2ce1 100644 --- a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + cloud_store_group_id + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + cloud_store_artifact_id + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12 + ${cloud_store_artifact_id} + false + + + cloud_store_version + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + @@ -49,12 +67,7 @@ - - - H/15 * * * * - false - - + diff --git a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml index df82bece0b..f8041941e2 100644 --- a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + cloud_store_group_id + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + cloud_store_artifact_id + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12 + ${cloud_store_artifact_id} + false + + + cloud_store_version + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml new file mode 100644 index 0000000000..af888dbe75 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml @@ -0,0 +1,149 @@ + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + 10 + -1 + 2 + + + + + false + false + + + + + private_branch + + choice-parameter-2544395024638227 + 1 + + true + + + + true + + + InquiryFlinkJob + Deploy/dev/KnowledgePlatform/InquiryFlinkJob + + + ET_FORMATTED_HTML + true + + + inquiry_release_tag + <font style="color:dimgray;font-size:14px;"><b> +<li>To build from a tag, use refs/tags/github_tag</li> +<li>To build from a branch, use refs/heads/github_branch</li> +<li>The default value of ${inquiry_pipeline_branch_or_tag} will be the release / tag version set in global configuration</li> +<li>To build from a differnt branch, replace the ${inquiry_pipeline_branch_or_tag} with your branch</li> +</b></font> + refs/heads/${inquiry_pipeline_branch_or_tag} + false + + + image_tag + <font color=red size=2><b>CAUTION: If the value is blank, image tag will be taken from the latest metadata.json.</b></font> + + false + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + ArtifactUpload/dev/KnowledgePlatform/InquiryFlinkJob + false + + + job_names_to_deploy + <font color=green size=2><b>Choose the job names to deploy. Multi-selection is available.</b></font> + choice-parameter-1273072434092073 + 1 + + true + + + + InquiryFlinkJob + Deploy/dev/KnowledgePlatform/InquiryFlinkJob + + + PT_MULTI_SELECT + false + 1 + + + + + 0 + 0 + + false + project + false + + + + + + + + + + 2 + + + https://github.com/Sunbird-inQuiry/data-pipeline.git + + + + + ${inquiry_release_tag} + + + false + + + + true + false + + 0 + false + + + + kubernetes/pipelines/deploy/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/jobs/Cassandra/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/jobs/Cassandra/config.xml index 1e9db63132..3304f15fad 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/jobs/Cassandra/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/jobs/Cassandra/config.xml @@ -1,219 +1,261 @@ - - - false - - - - -1 - 10 - -1 - 2 - - - - - false - false - - - - - absolute_job_path - <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> - ArtifactUpload/dev/Core/Cassandra - false - - - artifact_source - <font color=dimgray size=2><b> -ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrifact from Jenkins job.</b></font> - choice-parameter-9600649228560 - 1 - - true - - - - Cassandra - Deploy/dev/Kubernetes/Cassandra - - - PT_SINGLE_SELECT - false - 1 - - - build_number - - choice-parameter-9600651313765 - 1 - - true - - - - Cassandra - Deploy/dev/Kubernetes/Cassandra - - artifact_source - ET_FORMATTED_HTML - true - - - artifact_version - - choice-parameter-9600653373369 - 1 - - true - - - - Cassandra - Deploy/dev/Kubernetes/Cassandra - - artifact_source - ET_FORMATTED_HTML - true - - - private_branch - - choice-parameter-2544395024638227 - 1 - - true - - - - true - - - Cassandra - Deploy/dev/Kubernetes/Cassandra - - - ET_FORMATTED_HTML - true - - - branch_or_tag - - choice-parameter-2620434998790477 - 1 - + true + + + + Cassandra + Deploy/dev/Kubernetes/Cassandra + + + PT_SINGLE_SELECT + false + 1 + + + build_number + choice-parameter-591902397549104 + 1 + + true + + + + Cassandra + Deploy/dev/Kubernetes/Cassandra + + artifact_source + ET_FORMATTED_HTML + true + + + artifact_version + choice-parameter-591902398907654 + 1 + + true + + + + Cassandra + Deploy/dev/Kubernetes/Cassandra + + artifact_source + ET_FORMATTED_HTML + false + + + private_branch + choice-parameter-2544395024638227 + 1 + + true + + + + true + + + Cassandra + Deploy/dev/Kubernetes/Cassandra + + + ET_FORMATTED_HTML + true + + + branch_or_tag + choice-parameter-2620434998790477 + 1 + - true - - - - true - - - Cassandra - Deploy/dev/Kubernetes/Cassandra - - - ET_FORMATTED_HTML - true - - - - - 0 - 0 - - false - project - false - - - - - - - - ArtifactUpload/dev/Core/Cassandra - - SUCCESS - 0 - BLUE - true - - - - - - - - 2 - - - https://github.com/project-sunbird/sunbird-devops.git - - - - - ${branch_or_tag} - - - false - - - - true - false - - 0 - false - - - - pipelines/deploy/cassandra-deploy/Jenkinsfile - false - - - false + } + else + return """<b>This parameter is not used</b>""" + true + + + + true + + + Cassandra + Deploy/dev/Kubernetes/Cassandra + + + ET_FORMATTED_HTML + true + + + script_repo + Enter the repo url from which keyspace script folder to be uploaded + https://github.com/Sunbird-Lern/sunbird-utils.git + false + + + script_repo_branch_or_tag + Enter the branch or tag for keyspace script repo + ${branch_or_tag} + false + + + cassandra_keyspace_to_migrate + choice-parameter-598043621453772 + 1 + + true + + + + Cassandra + Deploy/dev/Kubernetes/Cassandra + + + PT_SINGLE_SELECT + false + 1 + + + + + 0 + 0 + + false + project + false + + + + + + + + ArtifactUpload/dev/Core/Cassandra + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${public_repo_branch} + + + false + + + + true + false + + 0 + false + + + + pipelines/deploy/cassandra-deploy/Jenkinsfile + false + + + false \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/jobs/InquiryUploadSchema/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/jobs/InquiryUploadSchema/config.xml new file mode 100644 index 0000000000..1735ad1e56 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/jobs/InquiryUploadSchema/config.xml @@ -0,0 +1,139 @@ + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + + -1 + -1 + -1 + 5 + + + + false + + + false + false + + + + + private_branch + choice-parameter-2544395024638227 + 1 + + true + + + + true + + + InquiryUploadSchema + Deploy/dev/Kubernetes/InquiryUploadSchema + + + ET_FORMATTED_HTML + true + + + branch_or_tag + choice-parameter-2620434998790477 + 1 + + true + + + + true + + + InquiryUploadSchema + Deploy/dev/Kubernetes/InquiryUploadSchema + + + ET_FORMATTED_HTML + true + + + schema_repo + Enter the repo url from which schema folder to be uploaded + https://github.com/Sunbird-inQuiry/inquiry-api-service.git + false + + + schema_repo_branch_or_tag + Enter the branch or tag for schema repo + false + + + + + 0 + 0 + + false + project + false + + + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/upload/schema/inquiry/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml index 859b804b88..f669caa862 100644 --- a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml @@ -26,7 +26,7 @@ @@ -114,4 +114,4 @@ return """<b>This parameter is not used</b>""" false - \ No newline at end of file + diff --git a/kubernetes/ansible/bootstrap.yaml b/kubernetes/ansible/bootstrap.yaml index 0fa7e0acfe..72f1beacff 100644 --- a/kubernetes/ansible/bootstrap.yaml +++ b/kubernetes/ansible/bootstrap.yaml @@ -39,7 +39,7 @@ - name: "Tagging {{ namespace }} to enable istio" shell: "kubectl label namespaces {{ namespace }} istio-injection=enabled --overwrite" - name: Creating docker secrets - shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ namespace }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password {{ vault_docker_registry_password }} --dry-run=client -o=yaml | kubectl apply -f -" + shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ namespace }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password '{{ vault_docker_registry_password }}' --dry-run=client -o=yaml | kubectl apply -f -" - name: Installing reloader for configmaps reload shell: helm upgrade --install --atomic reloader ../helm_charts/core/reloader --namespace "{{ namespace }}" -f /tmp/helm_vars.yaml roles: diff --git a/kubernetes/ansible/bootstrap_minimal.yaml b/kubernetes/ansible/bootstrap_minimal.yaml index 44170246ae..cdb145423e 100644 --- a/kubernetes/ansible/bootstrap_minimal.yaml +++ b/kubernetes/ansible/bootstrap_minimal.yaml @@ -30,7 +30,7 @@ - logging ignore_errors: yes - name: Creating docker secrets - shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password {{ vault_docker_registry_password }} --dry-run=client -o=yaml | kubectl apply -f -" + shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password '{{ vault_docker_registry_password }}' --dry-run=client -o=yaml | kubectl apply -f -" when: imagepullsecrets|length > 0 with_items: - "{{ bootstrap_namespace.split(',') }}" diff --git a/kubernetes/ansible/namespace.yml b/kubernetes/ansible/namespace.yml index bc62234c2c..bc38fab7d8 100644 --- a/kubernetes/ansible/namespace.yml +++ b/kubernetes/ansible/namespace.yml @@ -28,7 +28,7 @@ - "{{ bootstrap_namespace.split(',') }}" ignore_errors: yes - name: Creating docker secrets - shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password {{ vault_docker_registry_password }} --dry-run=client -o=yaml | kubectl apply -f -" + shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password '{{ vault_docker_registry_password }}' --dry-run=client -o=yaml | kubectl apply -f -" when: imagepullsecrets|length > 0 with_items: - "{{ bootstrap_namespace.split(',') }}" diff --git a/kubernetes/helm_charts/core/analytics/templates/deployment.yaml b/kubernetes/helm_charts/core/analytics/templates/deployment.yaml index 0926360f76..57198cb77b 100644 --- a/kubernetes/helm_charts/core/analytics/templates/deployment.yaml +++ b/kubernetes/helm_charts/core/analytics/templates/deployment.yaml @@ -35,14 +35,14 @@ spec: value: {{ .Values.env.min_heap | quote }} - name: MAX_HEAP value: {{ .Values.env.max_heap | quote }} - - name: azure_storage_secret - value: {{ .Values.env.azure_private_account_secret | quote }} - - name: azure_storage_key - value: {{ .Values.env.azure_private_account_name | quote }} - - name: public_azure_storage_secret - value: {{ .Values.env.azure_public_account_secret | quote }} - - name: public_azure_storage_key - value: {{ .Values.env.azure_public_account_name | quote }} + - name: cloud_storage_secret + value: {{ .Values.env.cloud_private_account_secret | quote }} + - name: cloud_storage_key + value: {{ .Values.env.cloud_private_account_name | quote }} + - name: public_cloud_storage_secret + value: {{ .Values.env.cloud_public_account_secret | quote }} + - name: public_cloud_storage_key + value: {{ .Values.env.cloud_public_account_name | quote }} - name: _JAVA_OPTIONS value: -Dlog4j2.formatMsgNoLookups=true envFrom: diff --git a/kubernetes/helm_charts/core/analytics/values.j2 b/kubernetes/helm_charts/core/analytics/values.j2 index 277f840947..5aae4e1daa 100644 --- a/kubernetes/helm_charts/core/analytics/values.j2 +++ b/kubernetes/helm_charts/core/analytics/values.j2 @@ -8,10 +8,10 @@ env: javaoptions: {{analytics_java_mem_limit|default('-Xmx600m')}} min_heap: {{analytics_min_heap_limit|default('-Xms1g')}} max_heap: {{analytics_max_heap_limit|default('-Xmx2g')}} - azure_private_account_secret: {{ cloud_private_storage_secret }} - azure_private_account_name: {{ cloud_private_storage_accountname }} - azure_public_account_secret: {{ cloud_public_storage_secret }} - azure_public_account_name: {{ cloud_public_storage_accountname }} + cloud_private_account_secret: {{ cloud_private_storage_secret }} + cloud_private_account_name: {{ cloud_private_storage_accountname }} + cloud_public_account_secret: {{ cloud_public_storage_secret }} + cloud_public_account_name: {{ cloud_public_storage_accountname }} replicaCount: {{analytics_replicacount|default(1)}} repository: {{analytics_repository|default('sunbird-analytics-service')}} image_tag: {{ image_tag }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/.helmignore b/kubernetes/helm_charts/core/ml-analytics-service/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/core/ml-analytics-service/Chart.yaml b/kubernetes/helm_charts/core/ml-analytics-service/Chart.yaml new file mode 100644 index 0000000000..eb2395299a --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: ml-analytics-service +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/_helpers.tpl b/kubernetes/helm_charts/core/ml-analytics-service/templates/_helpers.tpl new file mode 100644 index 0000000000..3f238f7ff7 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "ml-analytics-service.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ml-analytics-service.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ml-analytics-service.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "ml-analytics-service.labels" -}} +helm.sh/chart: {{ include "ml-analytics-service.chart" . }} +{{ include "ml-analytics-service.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "ml-analytics-service.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ml-analytics-service.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ml-analytics-service.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "ml-analytics-service.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/configmap.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/configmap.yaml new file mode 100644 index 0000000000..5f03f92832 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/configmap.yaml @@ -0,0 +1,11 @@ +--- +{{- $keys := .Files.Glob "keys/*" }} +{{ if $keys }} +apiVersion: v1 +kind: Secret +metadata: + name: ml-analytics-access-keys +type: Opaque +data: +{{ (.Files.Glob "keys/*").AsSecrets | indent 2 }} +{{ end }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml new file mode 100644 index 0000000000..d28628b7e5 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + volumes: + - name: {{ .Chart.Name }}-config + configMap: + name: {{ .Chart.Name }}-config + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + volumeMounts: + - name: {{ .Chart.Name }}-config + mountPath: /opt/sparkjobs/ml-analytics-service/config.ini + subPath: ml-analytics-service.conf diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/service.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/service.yaml new file mode 100644 index 0000000000..4b47ed5332 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/values.j2 b/kubernetes/helm_charts/core/ml-analytics-service/values.j2 new file mode 100644 index 0000000000..2a2a1c8638 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/values.j2 @@ -0,0 +1,21 @@ +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{ml_analytics_replicacount|default(1)}} +repository: {{ml_analytics_repository|default('ml-analytics-service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{ml_analytics_cpu_req|default('1000m')}} + memory: {{ml_analytics_mem_req|default('1000Mi')}} + limits: + cpu: {{ml_analytics_cpu_limit|default('2')}} + memory: {{ml_analytics_mem_limit|default('2048Mi')}} +network: + port: 7000 + targetport: 7000 +strategy: + type: RollingUpdate + maxsurge: {{ ml_core_maxsurge|default('25%') }} + maxunavailable: {{ ml_core_maxunavailable|default('25%') }} diff --git a/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml b/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml index b59cebe9a0..4cb69fd90f 100644 --- a/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml +++ b/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml @@ -40,7 +40,7 @@ data: listen 80; listen [::]:80; {{- if and .Values.nginx_private_ingress_ip (ne .Values.csp "oci") }} - server_name: {{ .Values.nginx_private_ingress_ip }}; + server_name {{ .Values.nginx_private_ingress_ip }}; {{- end }} resolver {{ .Values.kube_dns_ip }}; diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/templates/configMap.yaml b/kubernetes/helm_charts/core/nginx-public-ingress/templates/configMap.yaml index 0f7f0dcc16..3a04ccb80c 100644 --- a/kubernetes/helm_charts/core/nginx-public-ingress/templates/configMap.yaml +++ b/kubernetes/helm_charts/core/nginx-public-ingress/templates/configMap.yaml @@ -6,6 +6,10 @@ metadata: data: proxy-default.conf: | {{ .Values.proxyconfig | indent 4 }} +{{- if eq .Values.csp "oci" }} + cors-proxy-default.conf: | +{{ .Values.corsproxyconfig | indent 4 }} +{{- end }} compression.conf: | {{ .Values.compressionConfig | indent 4 }} diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 b/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 index e325f5d339..11e33d70c4 100644 --- a/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 +++ b/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 @@ -1,6 +1,7 @@ #jinja2:lstrip_blocks: True namespace: {{ namespace }} +csp: {{cloud_service_provider}} merge_domain_status: {{ merge_domain_status | lower }} service: annotations: {{nginx_public_ingress_service_annotations | d('') | to_json}} @@ -64,6 +65,74 @@ resources: repository: {{proxy_repository|default('proxy')}} image_tag: {{ image_tag }} +corsproxyconfig: |- + {% if proto=='https' %} + server { + if ($host = files.{{domain_name}}) { + return 301 https://$host$request_uri; + } + listen 80 ; + listen [::]:80 ; + server_name files.{{domain_name}}; + return 404; + } + {% endif %} + server { + {% if proto=='http' %} + listen 80; + listen [::]:80; + {% else %} + listen [::]:443 ssl ipv6only=on; + listen 443 ssl; + ssl_certificate /etc/secrets/site.crt; + ssl_certificate_key /etc/secrets/site.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA HIGH !RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS"; + {% endif %} + server_name files.{{domain_name}}; + client_max_body_size 0; + root /var/www/html; + resolver {{ kube_dns_ip }} valid=30s; + + location / { + # handle cors and allow all + if ($request_method = OPTIONS ) { + add_header Access-Control-Allow-Origin *; + add_header Access-Control-Allow-Methods "GET, OPTIONS, PATCH, POST, PUT, HEAD"; + add_header Access-Control-Allow-Headers "Access-Control-Allow-Origin, Authorization, Content-Type, user-id, Accept, Accept-Encoding, Accept-Language, Access-Control-Request-Headers, Access-Control-Request-Method, Cache-Control, DNT, User-Agent, X-Amz-Algorithm, X-Amz-Credential, X-Amz-Date, Amz-Expires, X-Amz-SignedHeaders, X-Amz-Signature, x-ms-blob-type"; + add_header Access-Control-Allow-Credentials "true"; + add_header Content-Length 0; + add_header Content-Type text/plain; + return 204; + } + + proxy_set_header Host "{{ cloud_storage_url | replace('https://', '') }}"; + # remove any CORS header from backend OSS S3 + proxy_hide_header Access-Control-Allow-Origin; + proxy_hide_header Access-Control-Allow-Methods; + proxy_hide_header Access-Control-Allow-Headers; + proxy_hide_header Access-Control-Allow-Credentials; + + # inject our own CORS header to allow what we wanted + add_header Access-Control-Allow-Credentials "true" always; + add_header Access-Control-Expose-Headers 'Content-Length,Content-Range,Connection,opc-client-info,opc-request-id' always; + add_header Access-Control-Allow-Origin * always; + add_header Access-Control-Allow-Methods "GET,OPTIONS,PATCH,POST,PUT,HEAD" always; + add_header Access-Control-Allow-Headers "Access-Control-Allow-Origin, Authorization, Content-Type, user-id, Accept,Accept-Encoding,Accept-Language, Access-Control-Request-Headers, Access-Control-Request-Method,Cache-Control,DNT,Host,Origin,Pragma,Referer,User-Agent, X-Amz-Algorithm, X-Amz-Credential, X-Amz-Date, Amz-Expires, X-Amz-SignedHeaders, X-Amz-Signature, x-ms-blob-type" always; + # + add_header Referer ""; + proxy_pass {{cloud_storage_url}}; + + # if get request, trim the query string + if ($request_method = GET ) { + proxy_pass {{cloud_storage_url}}$uri; + } + + + } + } + + proxyconfig: |- {% if proto=='https' %} server { diff --git a/pipelines/upload/schema/inquiry/Jenkinsfile b/pipelines/upload/schema/inquiry/Jenkinsfile new file mode 100644 index 0000000000..f37147a563 --- /dev/null +++ b/pipelines/upload/schema/inquiry/Jenkinsfile @@ -0,0 +1,55 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + } + + ansiColor('xterm') { + values = [:] + currentWs = sh(returnStdout: true, script: 'pwd').trim() + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + stage('deploy artifact'){ + sh """ + rm -rf schema_repo + git clone ${params.schema_repo} -b ${params.schema_repo_branch_or_tag} schema_repo + """ + ansiblePlaybook = "${currentWs}/ansible/kp_upload-schema.yml" + ansibleExtraArgs = "--extra-vars \" source_name=${currentWs}/schema_repo/schemas \" --vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = 'SUCCESS' + currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}, schema_repo_branch_or_tag: ${params.schema_repo_branch_or_tag}" + } + } + summary() + } + catch (err) { + currentBuild.result = 'FAILURE' + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} diff --git a/private_repo/ansible/inventory/dev/Core/common.yml b/private_repo/ansible/inventory/dev/Core/common.yml index 3386020d08..fe1f96338a 100644 --- a/private_repo/ansible/inventory/dev/Core/common.yml +++ b/private_repo/ansible/inventory/dev/Core/common.yml @@ -10,21 +10,25 @@ private_ingressgateway_ip: "" # your private kubernetes load balancer ip # Cloud Service Provider Variables # If cloud_service_provider is AWS then update with access key as value # Example: cloud_public_storage_accountname: "AKSHKSJHFJDHJDSHFKSD" +# If cloud_service_provider is OCI then update with access key as value +# Example: cloud_public_storage_accountname: "7855461d26e1d8f211a04ec59cfd3037c8d1c6dd" # If cloud_service_provider is gloud(GCP) then update with service account name # Example: cloud_public_storage_accountname: "cloudstorage-gcp-test.iam.gserviceaccount.com" # If cloud_service_provider is AZURE then update with stoage account name # Example: cloud_public_storage_accountname: "azurestotageaccount" cloud_public_storage_accountname: "" -# If cloud_service_provider is AWS then update with region +# If cloud_service_provider is AWS/OCI then update with region # Example: cloud_public_storage_region: us-east-1 cloud_public_storage_region: "" # If cloud_service_provider is gcp then update this variable with project id # Example: cloud_public_storage_project: "sunbird-gcp-test" cloud_public_storage_project: "" - +# If cloud_service_provider is OCI then update this variable with namespace value +# Example: cloud_public_storage_namespace: "apsyjjclx" +cloud_public_storage_namespace: "" # Create object storage for each below mentioned variables and update accordingly -# If cloud_service_provider is AWS update with bucket name +# If cloud_service_provider is AWS/OCI update with bucket name # If cloud_service_provider is gcloud(GCP) update with bucket name # If cloud_service_provider is AZURE update with container name # Example: cloud_storage_certqr_bucketname: "certqr-storage" @@ -96,6 +100,8 @@ cloud_storage_management_bucketname: "" # cloud_storage_url: https://storage.googleapis.com # AWS # cloud_storage_url: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" +# OCI +# cloud_storage_url: "https://{{ cloud_public_storage_namespace }}.compat.objectstorage.{{ cloud_public_storage_region }}.oraclecloud.com" # Azure cloud_storage_url: "https://{{ cloud_public_storage_accountname }}.blob.core.windows.net" @@ -106,6 +112,18 @@ cloud_storage_url: "https://{{ cloud_public_storage_accountname }}.blob.core.win # for google: # nginx_private_ingress_annotations: # cloud.google.com/load-balancer-type: "Internal" +# for OCI private +#nginx_private_ingress_annotations: +# service.beta.kubernetes.io/oci-load-balancer-internal: "true" +# service.beta.kubernetes.io/oci-load-balancer-subnet1: "ocid1.subnet.oc1.ap-hyderabad-1.xxxxxxxxxxxx" #subnet id +# for OCI public +# nginx public - oci specific +#nginx_public_ingress_service_annotations: +# oci.oraclecloud.com/load-balancer-type: "lb" +# service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" +# service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "100Mbps" +# service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "200Mbps" +# service.beta.kubernetes.io/oci-load-balancer-subnet1: "ocid1.subnet.oc1.ap-hyderabad-1.xxxxxxxxxxx" #subnet id # for azure: # nginx_private_ingress_annotations: # service.beta.kubernetes.io/azure-load-balancer-internal: "true" @@ -113,10 +131,26 @@ cloud_storage_url: "https://{{ cloud_public_storage_accountname }}.blob.core.win nginx_private_ingress_annotations: service.beta.kubernetes.io/azure-load-balancer-internal: "true" +# Media specific vars - OCI ( enable when cloud_service_provider is oci) +#oci_media_region: "{{ cloud_public_storage_region }}" +#oci_media_compartment: ocid1.compartment.oc1..xxxxxxxxxxxx # compartment id +#oci_media_namespace: "{{ cloud_public_storage_namespace }}" +#oci_media_source_bucket: "{{ cloud_storage_content_bucketname }}" +#oci_media_target_bucket: "{{ cloud_storage_content_bucketname }}" +#oci_media_prefix_input: "" +#oci_media_dist_channel_id: "" +#oci_media_work_flow_id: "" +#oci_media_stream_config_id: "" +#oci_media_gateway_domain: "" +#csp_migrator_router_parallelism: "" # ------------------------------------------------------------------------------------------------------------ # graylog_trusted_proxies: "1.2.3.4/21" # This should be the kubernetes nodes subnet CIDR range +### Android Mobile deeplink url - Upload the assetlinks.json file to the public object store (bucket) +## reference: https://project-sunbird.atlassian.net/wiki/spaces/SBDES/pages/3276865567/Deep+link+support+in+Android+12+devices +mobile_deeplink_url: "{{ cloud_storage_url }}/{{ cloud_storage_public_bucketname }}/Mobile-deeplink/assetlinks.json" + ## Optional learningservice_ip: "10.0.1.5" # Load balancer IP or server ip @@ -484,7 +518,23 @@ portal_redis_connection_string: "redis://:@{{ sunbird_redis_host }}:6379/3" # U #### ML Analytics Service ##### ml_analytics_cname_url: # Reference value - https://obj.dev.sunbirded.org/samiksha/ +## ML internal kong #### +internal_kong_postgres_host: +internal_kong_ssl: True +internal_kong_postgres_user: +internal_kong_postgres_password: +internal_kong_postgres_database: + ### Variables added as part of 6.0.0 release - https://github.com/Sunbird-Ed/ml-analytics-service/tree/release-6.0.0/release-notes/6.0.0.md ml_analytics_client_id : project-sunbird-dev-client # Keycloak client group ml_analytics_username : reportAdmin1 # Username of a user which has roles of "PROGRAM_MANAGER", "PROGRAM_DESIGNER", "REPORT_ADMIN", "REPORT_VIEWER" ml_analytics_createdBy : fb85a044-d9eb-479b-a55a-faf1bfaea14d # Unique system generated user UUID which is the same user as above +ml_container: samiksha +ml_kafka_topic_create: "" # set to true if ml related topics needs to be created +# provide the s3 compatible endpoint +# for AWS +# cloud_private_storage_account_endpoint: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" +# for OCI +#cloud_private_storage_account_endpoint: "https://.compat.objectstorage.{{cloud_public_storage_region}}.oraclecloud.com" +cloud_private_storage_account_endpoint: "{{ cloud_public_storage_endpoint }}" # Leave Blank for Azure +bb_name: "" #Leave Blank diff --git a/private_repo/ansible/inventory/dev/Core/secrets.yml b/private_repo/ansible/inventory/dev/Core/secrets.yml index 2e16859454..e511b72aac 100644 --- a/private_repo/ansible/inventory/dev/Core/secrets.yml +++ b/private_repo/ansible/inventory/dev/Core/secrets.yml @@ -5,7 +5,7 @@ core_vault_postgres_password: "change.postgres.password" # Update same core_vault_dp_pgdb_password: "change.postgres.password" #update same password in dp secrets dp_vault_pgdb_password variable core_vault_sunbird_azure_storage_key: "change.azure.storage.account.key" core_vault_docker_registry_url: "change.docker.url" # for docker hub "https://index.docker.io/v1" -core_vault_docker_registry_user: "change.docker.username" +core_vault_docker_registry_user: "change.docker.username" # For OCI oci_namespcae/oci_username core_vault_docker_registry_password: "change.docker.password" # Cloud Service Provider Secret Variables @@ -31,7 +31,7 @@ core_vault_docker_registry_password: "change.docker.password" # Example: cloud_management_storage_secret: "BfoOQwA6UFI7URwkFjkjdjsdhjhjhjsdhjfM//lsTbXDmHOFR5GqEwAwzyrKXOUDywojCpOC/g==" cloud_management_storage_secret: "" -# If cloud_service_provider is aws then update secret access key +# If cloud_service_provider is aws/oci then update secret access key # Example: cloud_public_storage_secret: "xkxjfjsdsfjdfjdhgjfhgjdhfgjdgjdfgdjgjdgdjhgjhd" # If cloud_service_provider is gcp then update with private-key which is in service account json file @@ -238,3 +238,7 @@ lp_vault_youtube_api_key: # youtube api token if you want ml_analytics_authorization_access_token : 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJkOTY3NzRjYzXXXXXXXXXXXXXQ4Y2RiOWQ2Mzg0OSJ9.n4hXxKxl_698yeZPSWhXeGvMOb4esfgPadIZe8jZ0Z4' # Bearer auth token which has access to APIs as defined in the release note ml_analytics_password : Test@123 # Password of a user which has roles of "PROGRAM_MANAGER", "PROGRAM_DESIGNER", "REPORT_ADMIN", "REPORT_VIEWER" matching with supplied username to variable ml_analytics_username ml_analytics_client_secret : fd241dce-4ZZ9-47e1-97cf-1c7de7a44216 # Keycloak client secret for creating tokens, mapped to client id supplied to variable ml_analytics_client_id + +#### ML Core Service ##### +ml_core_internal_access_token: 8c3a94f0934e01a4940a # THIS IS A REFERENCE VALUE ONLY.Secure token to access Admin-related APIs.The value of the internal access token can be generated by command - "openssl rand -hex 10" +ml_api_access_token: 8c3a94f0934e01a4940a # THIS IS A REFERENCE VALUE ONLY.Secure token to access Admin-related APIs.The value of the internal access token can be generated by command - "openssl rand -hex 10" diff --git a/private_repo/ansible/inventory/dev/DataPipeline/common.yml b/private_repo/ansible/inventory/dev/DataPipeline/common.yml index c5790ec42c..fa568ffdf2 100644 --- a/private_repo/ansible/inventory/dev/DataPipeline/common.yml +++ b/private_repo/ansible/inventory/dev/DataPipeline/common.yml @@ -6,25 +6,30 @@ domain_name: "" # your domain name like example.com dockerhub: "change.docker.url" # docker hub username or url incase of private registry private_ingressgateway_ip: "" # your private kubernetes load balancer ip search_lb_ip: # search service Load balancer IP +learningservice_ip: "10.0.1.5" # Load balancer IP or server ip # Cloud Service Provider Variables # If cloud_service_provider is AWS then update with access key as value # Example: cloud_public_storage_accountname: "AKSHKSJHFJDHJDSHFKSD" +# If cloud_service_provider is OCI then update with access key as value +# Example: cloud_public_storage_accountname: "7855461d26e1d8f211a04ec59cfd3037c8d1c6dd" # If cloud_service_provider is gloud(GCP) then update with service account name # Example: cloud_public_storage_accountname: "cloudstorage-gcp-test.iam.gserviceaccount.com" # If cloud_service_provider is AZURE then update with stoage account name # Example: cloud_public_storage_accountname: "azurestotageaccount" + cloud_public_storage_accountname: "" -# If cloud_service_provider is AWS then update with region +# If cloud_service_provider is AWS/OCI then update with region # Example: cloud_public_storage_region: us-east-1 cloud_public_storage_region: "" # If cloud_service_provider is gcp then update this variable with project id # Example: cloud_public_storage_project: "sunbird-gcp-test" cloud_public_storage_project: "" - - +# If cloud_service_provider is OCI then update this variable with namespace value +# Example: cloud_public_storage_namespace: "apsyjjclx" +cloud_public_storage_namespace: "" # Create object storage for each below mentioned variables and update accordingly -# If cloud_service_provider is AWS update with bucket name +# If cloud_service_provider is AWS/OCI update with bucket name # If cloud_service_provider is gcloud(GCP) update with bucket name # If cloud_service_provider is AZURE update with container name # Example: cloud_storage_certqr_bucketname: "certqr-storage" @@ -96,6 +101,8 @@ cloud_storage_management_bucketname: "" # cloud_storage_url: https://storage.googleapis.com # AWS # cloud_storage_url: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" +# OCI +# cloud_storage_url: "https://{{ cloud_public_storage_namespace }}.compat.objectstorage.{{ cloud_public_storage_region }}.oraclecloud.com" # Azure cloud_storage_url: "https://{{ cloud_public_storage_accountname }}.blob.core.windows.net" @@ -130,6 +137,11 @@ postgres: db_port: 5432 db_admin_user: postgres db_admin_password: "{{dp_vault_pgdb_admin_password}}" + dp_ssl: false + dp_ssl_mode: require + dp_sslfactory: org.postgresql.ssl.NonValidatingFactory + dp_db_name: analytics + spark_db_name: analytics druid_postgres_user: druid # Do not change this imagepullsecrets: "{{env}}registrysecret" # kubernetes imagePullSecrets @@ -178,6 +190,11 @@ cloudstorage_base_path: "{{ cloud_storage_url }}" valid_cloudstorage_base_urls: '["{{ cloud_storage_url }}"]' cloudstorage_relative_path_prefix: "CONTENT_STORAGE_BASE_PATH" +cloud_storage_telemetry_type: "{{ cloud_service_provider }}" +druid_storage_type: +spark_bds_url: +druid_cloud_storage_pathstyle_access: true +spark_version: # overide this var if you need a different version of spark. Ex: spark_version: 3.1.3 # The below sets the kafka topics retention time to 1 day, if you use the defaults from the public repo, it will be 7 days # If you want to retain the topics for 7 days, remove the below sections completely # Ensure you have atleast 1 TB of disk to retain data for 7 days @@ -295,4 +312,4 @@ processing_kafka_overriden_topics: replication_factor: 1 # graylog -send_logs_to_graylog: true # filebeat agents will send logs to graylog instead of ES \ No newline at end of file +send_logs_to_graylog: true # filebeat agents will send logs to graylog instead of ES diff --git a/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml b/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml index ce4e93b37e..5c13f9d9b9 100644 --- a/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml +++ b/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml @@ -5,16 +5,18 @@ cloud_service_provider: "" # Your cloud service provider name. Supported v dockerhub: "change.docker.url" # docker hub username or url incase of private registry private_ingressgateway_ip: "" # your private kubernetes load balancer ip domain_name: "" # your domain name like example.com +kp_print_service_base_url: "http://{{private_ingressgateway_ip}}/print" +learningservice_ip: "10.0.1.5" # Load balancer IP or server ip # Cloud Service Provider Variables -# If cloud_service_provider is AWS then update with access key as value +# If cloud_service_provider is AWS/OCI then update with access key as value # Example: cloud_public_storage_accountname: "AKSHKSJHFJDHJDSHFKSD" # If cloud_service_provider is gloud(GCP) then update with service account name # Example: cloud_public_storage_accountname: "cloudstorage-gcp-test.iam.gserviceaccount.com" # If cloud_service_provider is AZURE then update with stoage account name # Example: cloud_public_storage_accountname: "azurestotageaccount" cloud_public_storage_accountname: "" -# If cloud_service_provider is AWS then update with region +# If cloud_service_provider is AWS/OCI then update with region # Example: cloud_public_storage_region: us-east-1 cloud_public_storage_region: "" # If cloud_service_provider is gcp then update this variable with project id @@ -26,7 +28,7 @@ cloud_public_storage_namespace: "" # Create object storage for each below mentioned variables and update accordingly -# If cloud_service_provider is AWS update with bucket name +# If cloud_service_provider is AWS/OCI update with bucket name # If cloud_service_provider is gcloud(GCP) update with bucket name # If cloud_service_provider is AZURE update with container name # Example: cloud_storage_certqr_bucketname: "certqr-storage" @@ -98,6 +100,8 @@ cloud_storage_management_bucketname: "" # cloud_storage_url: https://storage.googleapis.com # AWS # cloud_storage_url: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" +# OCI +# cloud_storage_url: "https://{{ cloud_public_storage_namespace }}.compat.objectstorage.{{ cloud_public_storage_region }}.oraclecloud.com" # Azure cloud_storage_url: "https://{{ cloud_public_storage_accountname }}.blob.core.windows.net" @@ -129,8 +133,11 @@ kubeconfig_path: /var/lib/jenkins/secrets/k8s.yaml # kubeconfig file path # cloud_public_storage_endpoint: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" # for OCI #cloud_public_storage_endpoint: "https://.compat.objectstorage.{{cloud_public_storage_region}}.oraclecloud.com" - -cloud_public_storage_endpoint: "" +#oci_flink_s3_storage_endpoint: "{{ cloud_public_storage_endpoint }}" +#cloudstorage_sdk_endpoint: "{{ cloud_public_storage_endpoint }}" +#s3_region: "{{ cloud_public_storage_region }}" +#cloud_storage_proxy_host: "{{cloud_storage_url}}" +#flink_container_name: "{{ cloud_storage_flink_bucketname }}" # Update below vars if seperate object storage is required cloud_private_storage_accountname: "{{ cloud_public_storage_accountname }}" @@ -163,13 +170,26 @@ cloud_artifact_storage_namespace: "{{ cloud_public_storage_namespace }}" # cloud_storage_neo4jbackup_bucketname: "" # cloud_storage_redisbackup_bucketname: "" +# Media specific vars - OCI ( enable when cloud_service_provider is oci) +#oci_media_region: "{{ cloud_public_storage_region }}" +#oci_media_compartment: ocid1.compartment.oc1..xxxxxxxxxxxx # compartment id +#oci_media_namespace: "{{ cloud_public_storage_namespace }}" +#oci_media_source_bucket: "{{ cloud_storage_content_bucketname }}" +#oci_media_target_bucket: "{{ cloud_storage_content_bucketname }}" +#oci_media_prefix_input: "" +#oci_media_dist_channel_id: "" +#oci_media_work_flow_id: "" +#oci_media_stream_config_id: "" +#oci_media_gateway_domain: "" +#csp_migrator_router_parallelism: "" + # Building block vars cloud_storage_base_url: "{{ cloud_storage_url }}" cloudstorage_base_path: "{{ cloud_storage_url }}" valid_cloudstorage_base_urls: '["{{ cloud_storage_url }}"]' cloudstorage_relative_path_prefix: "CONTENT_STORAGE_BASE_PATH" -cloud_storage_pathstyle_access: false - +cloud_storage_pathstyle_access: true +cloud_storage_cname_url: "{{ cloud_storage_url }}" # overide if you have seperate url for cname ### Lern BB - Adding Lern specific vars here. In future if we want to move it to seperate folder this can be used as the starting point # Mandatorty variables - DO NOT LEAVE ANYTHING BLANK # @@ -213,4 +233,10 @@ postgres: db_admin_password: "{{dp_vault_pgdb_admin_password}}" # graylog -send_logs_to_graylog: true # filebeat agents will send logs to graylog instead of ES \ No newline at end of file +send_logs_to_graylog: true # filebeat agents will send logs to graylog instead of ES + +druid_storage_type: "" # Your cloud service provider name. Supported values are aws, azure, gcloud + +# Optional variables - Can be left blank +cloud_storage_report_verfication_bucketname: "" +dp_storage_endpoint_config: ""