diff --git a/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js b/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js index 5b4b6c6125..1460374799 100644 --- a/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js +++ b/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js @@ -3159,6 +3159,7 @@ var handleGoogleAuthEvent = () => { redirectToLib(); } }; + var redirectToPortal = (redirectUrlPath) => { // redirectUrlPath for sso and self signUp const curUrlObj = window.location; var redirect_uri = getValueFromSession('redirect_uri'); @@ -3171,9 +3172,7 @@ var redirectToPortal = (redirectUrlPath) => { // redirectUrlPath for sso and sel const redirect_uriLocation = new URL(redirect_uri); if (client_id === 'android' || client_id === 'desktop') { window.location.href = sessionUrlObj.protocol + '//' + sessionUrlObj.host + redirectUrlPath + updatedQuery; - } else if(client_id === 'portal' && - redirectUrlPath === '/sign-in/sso/select-org' && - (redirect_uri.includes('dock.sunbirded.org') || redirect_uri.includes('dockstaging.sunbirded.org'))) { + } else if(client_id === 'portal' && redirectUrlPath === '/sign-in/sso/select-org' && (sessionUrlObj.host !== redirect_uriLocation.host)) { window.location.href = sessionUrlObj.protocol + '//' + sessionUrlObj.host + redirectUrlPath + updatedQuery; } else { window.location.href = redirect_uriLocation.protocol + '//' + redirect_uriLocation.host + diff --git a/ansible/roles/cassandra/defaults/main.yml b/ansible/roles/cassandra/defaults/main.yml index 33c02c17bf..8f1f0f7951 100644 --- a/ansible/roles/cassandra/defaults/main.yml +++ b/ansible/roles/cassandra/defaults/main.yml @@ -1,5 +1,5 @@ user_home: "/home/{{ ansible_ssh_user }}/" -cassandra_repo: 'deb http://www.apache.org/dist/cassandra/debian 311x main' -cassandra_repo_key: 'https://www.apache.org/dist/cassandra/KEYS' +cassandra_repo: 'deb https://debian.cassandra.apache.org 311x main' +cassandra_repo_key: 'https://downloads.apache.org/cassandra/KEYS' resource_crunch: "yes" # make this value yes if heap size should be quarter of the Server Memory when you are deploying multiple databases, else make this value no to have 50% of the value as per the best practice of cassandra cassandra_home_directory: "/var/lib/cassandra" diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 6090bfff0f..837c208319 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -8237,6 +8237,27 @@ kong_apis: config.required: false config.enabled: false +- name: uploadFileToCloud + uris: "{{ cloud_service_prefix }}/mlcore/v1/files/upload" + upstream_url: "{{ ml_core_service_url }}/v1/cloud-services/files/upload" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - cloudUrlsCreate + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + - name: listPlatformPrograms uris: "{{ user_extension_prefix }}/mlcore/v1/programsByPlatformRoles" upstream_url: "{{ ml_core_service_url }}/v1/user-extension/programsByPlatformRoles" diff --git a/ansible/roles/log-es6/tasks/plugins/repository-oci.yml b/ansible/roles/log-es6/tasks/plugins/repository-oci.yml index 02a2b45f45..3018905845 100644 --- a/ansible/roles/log-es6/tasks/plugins/repository-oci.yml +++ b/ansible/roles/log-es6/tasks/plugins/repository-oci.yml @@ -4,14 +4,14 @@ shell: echo "{{ cloud_management_storage_accountname }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.access_key no_log: True environment: - ES_PATH_CONF: "{{ conf_dir }}" + ES_PATH_CONF: "{{ es_conf_dir }}" - name: Add default oci account key for backups become: yes shell: echo "{{ cloud_management_storage_secret }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.secret_key no_log: True environment: - ES_PATH_CONF: "{{ conf_dir }}" + ES_PATH_CONF: "{{ es_conf_dir }}" - name: Add default oci endpoint for backups @@ -19,18 +19,18 @@ shell: echo "{{ cloud_management_storage_endpoint }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.endpoint no_log: True environment: - ES_PATH_CONF: "{{ conf_dir }}" + ES_PATH_CONF: "{{ es_conf_dir }}" - name: Add default oci region for backups become: yes shell: echo "{{ cloud_management_storage_region }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.region no_log: True environment: - ES_PATH_CONF: "{{ conf_dir }}" + ES_PATH_CONF: "{{ es_conf_dir }}" - name: Add default path-style access for backups become: yes shell: echo "{{ cloud_management_storage_path_style_access }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.path_style_access no_log: True environment: - ES_PATH_CONF: "{{ conf_dir }}" \ No newline at end of file + ES_PATH_CONF: "{{ es_conf_dir }}" diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index 15ef432e43..b85fe27059 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -133,3 +133,9 @@ ml_analytics_reports_store: "{{ cloud_service_provider }}" ml_analytics_reports_container: "{{ cloud_storage_privatereports_bucketname }}" ml_analytics_driver_memory: "{{ ml_analytics_default_driver_memory | default('5g') }}" ml_analytics_executor_memory: "{{ ml_analytics_default_executor_memory | default('5g') }}" +ml_analytics_batch_cron_minute: "{{ ml_batch_cron_minute | default('30') }}" +ml_analytics_batch_cron_hour: "{{ ml_batch_cron_hour | default('18') }}" +ml_analytics_nvsk_cron_minute: "{{ ml_nvsk_cron_minute | default('30') }}" +ml_analytics_nvsk_cron_hour: "{{ ml_nvsk_cron_hour | default('7') }}" +ml_analytics_nvsk_cron_weekday: "{{ ml_nvsk_cron_weekday | default('4') }}" +ml_analytics_faust_applications_list: "'observations/py_observation_streaming observations/', 'observations/py_observation_evidence_streaming observations/', 'survey/py_survey_streaming survey/', 'survey/py_survey_evidence_streaming survey/'" \ No newline at end of file diff --git a/ansible/roles/ml-analytics-service/tasks/main.yml b/ansible/roles/ml-analytics-service/tasks/main.yml index 6f37d6b3ea..7fbd3fe1ba 100755 --- a/ansible/roles/ml-analytics-service/tasks/main.yml +++ b/ansible/roles/ml-analytics-service/tasks/main.yml @@ -26,6 +26,7 @@ - python3-virtualenv - zip - unzip + - acl state: present - name: CHANGE THE OWNERSHIP FOR THIS {{ BASEPATH }} DIRECTORY @@ -135,15 +136,15 @@ cron: name: "Run Batch Ingestion Job" user: "{{ USER }}" - minute: "30" - hour: "18" + minute: "{{ ml_analytics_batch_cron_minute }}" + hour: "{{ ml_analytics_batch_cron_hour }}" job: "{{ BASEPATH }}/ml-analytics-service/run.sh > {{ BASEPATH }}/ml-analytics-service/crontab_job.log" - name: CREATE THE CRON NVSK Data Upload cron: name: "NVSK Data Upload JOB" user: "{{ USER }}" - minute: "30" - hour: "7" - weekday: "4" + minute: "{{ ml_analytics_nvsk_cron_minute }}" + hour: "{{ ml_analytics_nvsk_cron_hour }}" + weekday: "{{ ml_analytics_nvsk_cron_weekday }}" job: "{{ BASEPATH }}/ml-analytics-service/run_weekly.sh > {{ BASEPATH }}/ml-analytics-service/nvsk_data_weekly.logs" diff --git a/ansible/roles/ml-analytics-service/templates/config.j2 b/ansible/roles/ml-analytics-service/templates/config.j2 index 935085d36a..201e9ab8b5 100644 --- a/ansible/roles/ml-analytics-service/templates/config.j2 +++ b/ansible/roles/ml-analytics-service/templates/config.j2 @@ -263,3 +263,8 @@ store = {{ ml_analytics_reports_store }} [ML_CORE_SERVICE_URL] url = {{ ml_analytics_core_service }} + + +[SERVICES] + +faust_applications_list = {{ ml_analytics_faust_applications_list }} diff --git a/ansible/roles/mongodb-cluster/templates/mongod.conf.j2 b/ansible/roles/mongodb-cluster/templates/mongod.conf.j2 index c896d7599a..fabdee7da7 100644 --- a/ansible/roles/mongodb-cluster/templates/mongod.conf.j2 +++ b/ansible/roles/mongodb-cluster/templates/mongod.conf.j2 @@ -106,7 +106,7 @@ operationProfiling: {% endfor %} {% endif %} -{% if mongodb_major_version is version("4.0", ">=") -%} +{# {% if mongodb_major_version is version("4.0", ">=") -%} cloud: monitoring: free: @@ -116,7 +116,7 @@ cloud: {{ item }} {% endfor %} {% endif %} -{% endif %} +{% endif %} #} {% if mongodb_set_parameters -%} setParameter: diff --git a/ansible/roles/stack-sunbird/defaults/main.yml b/ansible/roles/stack-sunbird/defaults/main.yml index 581f28458b..0c0a678be5 100644 --- a/ansible/roles/stack-sunbird/defaults/main.yml +++ b/ansible/roles/stack-sunbird/defaults/main.yml @@ -268,6 +268,7 @@ service_env: ml-reports-service: ../../../../ansible/roles/stack-sunbird/templates/ml-reports-service.env ml-core-service: ../../../../ansible/roles/stack-sunbird/templates/ml-core-service.env ml-projects-service: ../../../../ansible/roles/stack-sunbird/templates/ml-projects-service.env + ml-analytics-service: ../../../../ansible/roles/stack-sunbird/templates/ml-analytics-service.conf inbound: ../../../../ansible/roles/stack-sunbird/templates/inbound.env outbound: - ../../../../ansible/roles/stack-sunbird/templates/outbound.env diff --git a/ansible/roles/stack-sunbird/templates/analytics_api_service.conf b/ansible/roles/stack-sunbird/templates/analytics_api_service.conf index 8bdcd63467..b9a8c838c8 100644 --- a/ansible/roles/stack-sunbird/templates/analytics_api_service.conf +++ b/ansible/roles/stack-sunbird/templates/analytics_api_service.conf @@ -201,10 +201,10 @@ public { } } cloud_storage_type="{{cloud_service_provider}}" -storage.key.config="{{cloud_public_storage_accountname}}" -storage.secret.config="{{cloud_public_storage_secret}}" -public.storage.key.config="{{cloud_public_storage_accountname}}" -public.storage.secret.config="{{cloud_public_storage_secret}}" +storage.key.config="cloud_storage_key" +storage.secret.config="cloud_storage_secret" +public.storage.key.config="public_cloud_storage_key" +public.storage.secret.config="public_cloud_storage_secret" cloud_storage_endpoint_with_protocol="{{cloud_storage_url}}" metrics.time.interval.min=30 cache.refresh.time.interval.min=1440 diff --git a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf new file mode 100644 index 0000000000..707dc83e6f --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf @@ -0,0 +1,294 @@ +[ML_APP_NAME] + +survey_app = {{ ml_survey_app_name | default('sunbirdsurvey') }} + +integrated_app = {{ ml_integrated_app_name | default('sunbird') }} + +integrated_portal = {{ ml_integrated_portal | default('dev.sunbird.portal') }} + + +[API_ENDPOINTS] + +base_url = https://{{ domain_name }}/ + + +[API_HEADERS] + +content_type = application/json + +authorization = {{ml_api_auth_token | default('sunbird_api_auth_token')}} + +internal_access_token = {{ml_api_access_token | default('ml_core_internal_access_token')}} + +authorization_access_token = {{ ml_analytics_authorization_access_token }} + +[ML_SURVEY_SERVICE_URL] + +url = http://{{private_ingressgateway_ip}}/ml-survey + +entity_related_end_point = assessment/api/v1/entities/relatedEntities/ + +user_profile_end_point = assessment/api/v1/userExtension/getProfile/ + +evidence_base_url = {{ cloud_storage_url }}/{{ ml_analytics_public_storage }}/ + + +[MONGO] + +url = mongodb://{{ml_mongodb_host | default(groups['mongo_master'][0]+':27017')}} + +database_name = {{ml_mongodb | default('ml-survey')}} + + +# ------ Mongo Collections ------- # +observation_sub_collection = observationSubmissions + +solutions_collection = solutions + +observations_collection = observations + +entity_type_collection = entityTypes + +questions_collection = questions + +criteria_collection = criteria + +entities_collection = entities + +programs_collection = programs + +user_roles_collection = userRoles + +criteria_questions_collection = criteriaQuestions + +projects_collection = projects + +survey_submissions_collection = surveySubmissions + +survey_collection = surveys + +[DRUID] + +metadata_url = http://{{groups['druid'][0]}}:8081/druid/coordinator/v1/datasources/ + +batch_url = http://{{groups['druid'][0]}}:8081/druid/indexer/v1/task + +metadata_rollup_url = http://{{groups['druid'][0]}}:8081/druid/coordinator/v1/datasources/ + +batch_rollup_url = http://{{groups['druid'][0]}}:8081/druid/indexer/v1/task + +urlQuery = http://{{groups['druid'][0]}}:8082/druid/v2?pretty + +datasource_url = http://{{groups['ml-raw-broker'][0]}}:{{ml_raw_broker_port}}/druid/v2/datasources + +sql_url = http://{{groups['ml-raw-broker'][0]}}:{{ml_raw_broker_port}}/druid/v2/sql/ + +coordinator_url = http://{{groups['ml-raw-coordinator'][0]}}:{{ml_raw_coordinator_port}}/status/health + +overload_url = http://{{groups['ml-raw-overload'][0]}}:{{ml_raw_overload_port}}/status/health + +historical_url = http://{{groups['ml-raw-historical'][0]}}:{{ml_raw_historical_port}}/status/health + +observation_status_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_code","school_externalId","state_name","state_code","state_externalId","district_name","district_code","district_externalId","block_name","block_code","block_externalId","cluster_name","cluster_code","cluster_externalId","completedDate","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name","ecm_marked_na","board_name","updatedAt","organisation_id","user_type","observed_school_name","observed_school_id","observed_school_code","observed_state_name","observed_state_id","observed_state_code","observed_district_name","observed_district_id","observed_district_code","observed_block_name","observed_block_id","observed_block_code","observed_cluster_name","observed_cluster_id","observed_cluster_code","isRubricDriven","criteriaLevelReport"]},"metricsSpec":[]}}}' + +project_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-project","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"auto"},"dimensionsSpec":{"dimensions":[]},"metricsSpec":[]}}}' + +ml_distinctCnt_obs_status_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' + +ml_distinctCnt_obs_domain_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' + +ml_distinctCnt_obs_domain_criteria_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain-criteria","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"string","name":"criteria_name"},{"type":"string","name":"criteria_score"},{"type":"string","name":"criteria_id"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' + +ml_distinctCnt_projects_status_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"project_title"},{"type":"string","name":"solution_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"long","name":"no_of_imp_with_evidence_inprogress"},{"type":"string","name":"time_stamp"},{"type":"long","name":"no_of_certificate_issued"}]},"metricsSpec":[]}}}' + +ml_distinctCnt_prglevel_projects_status_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-programLevel-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"},{"type":"long","name":"no_of_certificate_issued"}]},"metricsSpec":[]}}}' + +ml_distinctCnt_survey_status_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-surveydistinctCount-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"survey_name"},{"type":"string","name":"survey_id"},{"type":"string","name":"submission_status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_submissions"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' + +observation_status_rollup_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-observation-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["status","user_id","solution_id","submission_id","entity_name","completedDate","program_id","private_program","solution_type","updatedAt","role_title","solution_name","program_name","channel","parent_channel","block_name","district_name","school_name","cluster_name","state_name","organisation_name","board_name","district_externalId","state_externalId","block_externalId","cluster_externalId","school_externalId","organisation_id",{"type":"long","name":"status_code"}]},"metricsSpec":[{"type":"count","name":"count"},{"type":"longSum","name":"sum___v","fieldName":"__v","expression":null},{"type":"HLLSketchBuild","name":"count_distinct_solution","fieldName":"solution_id","lgK":12,"tgtHllType":"HLL_4","round":false},{"type":"HLLSketchBuild","name":"count_distinct_submission_id","fieldName":"submission_id","lgK":12,"tgtHllType":"HLL_4","round":false},{"type":"HLLSketchBuild","name":"count_distinct_user_id","fieldName":"user_id","lgK":12,"tgtHllType":"HLL_4","round":false}]}}}' + +project_rollup_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"iso"},"dimensionsSpec":{"dimensions":["project_title","project_goal","area_of_improvement","status_of_project","tasks_name","tasks_status","designation","task_evidence_status","project_id","task_id","project_created_type","parent_channel","program_id","program_name","project_updated_date","createdBy","program_externalId","private_program","task_deleted_flag","project_terms_and_condition","state_externalId","block_externalId","district_externalId","cluster_externalId","school_externalId","state_name","block_name","district_name","cluster_name","school_name","board_name","organisation_name","solution_id","organisation_id",{"name":"status_code","type":"long"}]},"metricsSpec":[{"name":"count","type":"count"},{"name":"sum___v","type":"longSum","fieldName":"__v"},{"name":"sum_status_code","type":"longMax","fieldName":"status_code"},{"type":"HLLSketchBuild","name":"count_of_createBy","fieldName":"createdBy"},{"type":"HLLSketchBuild","name":"count_of_project_id","fieldName":"project_id"},{"type":"HLLSketchBuild","name":"count_of_solution_id","fieldName":"solution_id"},{"type":"HLLSketchBuild","name":"count_of_program_id","fieldName":"program_id"}]}}}' + +ml_survey_rollup_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel",{"type":"long","name":"status_code"}, "solution_name", "solution_id"]},"metricsSpec":[{"name":"count","type":"count"},{"name":"sum___v","type":"longSum","fieldName":"__v"},{"name":"sum_status_code","type":"longMax","fieldName":"status_code"},{"type":"HLLSketchBuild","name":"count_of_user_id","fieldName":"user_id"},{"type":"HLLSketchBuild","name":"count_of_survey_submission_id","fieldName":"survey_submission_id"},{"type":"HLLSketchBuild","name":"count_of_solution_id","fieldName":"solution_id"},{"type":"HLLSketchBuild","name":"count_of_program_id","fieldName":"program_id"}]}}}' + +survey_status_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel", "solution_name", "solution_id","private_program","state_code","school_code","district_code","block_code","cluster_code"]},"metricsSpec":[]}}}' + +observation_query_spec = '{"queryType":"scan","dataSource":"sl-observation","resultFormat":"list","columns":["completedDate","createdAt","createdBy","criteriaExternalId","criteriaId","criteriaName","entityType","entityTypeId","observationId","observationName","observationSubmissionId","questionAnswer","questionECM","questionExternalId","questionId","questionName","questionResponseLabel","questionResponseType","solutionExternalId","solutionId","solutionName","updatedAt","instanceParentId","instanceId","instanceParentResponsetype","instanceParentQuestion","questionSequenceByEcm","maxScore","minScore","percentageScore","pointsBasedScoreInParent","totalScore","scoreAchieved","totalpercentage","instanceParentExternalId","instanceParentEcmSequence","remarks","total_evidences","evidence_count","school","block","district","cluster","state","schoolName","blockName","districtName","clusterName","stateName","schoolExternalId","blockExternalId","districtExternalId","clusterExternalId","stateExternalId","schoolTypes","administrationTypes","instanceParentCriteriaId","instanceParentCriteriaExternalId","instanceParentCriteriaName","role_title","location_validated_with_geotag","distance_in_meters","entity","entityExternalId","entityName","isAPrivateProgram","programId","programName","programExternalId","questionResponseLabel_number","criteriaLevel","criteriaScore","submissionNumber","submissionTitle","channel","parent_channel","user_districtName","user_blockName","user_clusterName","appName","evidences","user_stateName","domainName","domainExternalId","childName","childType","childExternalid","level","criteriaDescription","programDescription","solutionDescription","label","imp_project_id","imp_project_title","imp_project_goal","imp_project_externalId","ancestorName","scoringSystem","domainLevel","domainScore","criteriaLevelReport","user_schoolName","user_schoolId","user_schoolUDISE_code","solution_type","organisation_name","user_boardName","district_externalId","state_externalId","block_externalId","cluster_externalId","organisation_id","user_type"],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]}' + +observation_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"entityType"},{"type":"string","name":"entityTypeId"},{"type":"string","name":"observationId"},{"type":"string","name":"observationName"},{"type":"string","name":"observationSubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"school"},{"type":"string","name":"block"},{"type":"string","name":"district"},{"type":"string","name":"cluster"},{"type":"string","name":"state"},{"type":"string","name":"schoolName"},{"type":"string","name":"blockName"},{"type":"string","name":"districtName"},{"type":"string","name":"clusterName"},{"type":"string","name":"stateName"},{"type":"string","name":"schoolExternalId"},{"type":"string","name":"blockExternalId"},{"type":"string","name":"districtExternalId"},{"type":"string","name":"clusterExternalId"},{"type":"string","name":"stateExternalId"},{"type":"string","name":"schoolTypes"},{"type":"string","name":"administrationTypes"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"role_title"},{"type":"string","name":"location_validated_with_geotag"},{"type":"string","name":"distance_in_meters"},{"type":"string","name":"entity"},{"type":"string","name":"entityExternalId"},{"type":"string","name":"entityName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"criteriaLevel"},{"type":"string","name":"criteriaScore"},{"type":"string","name":"submissionNumber"},{"type":"string","name":"submissionTitle"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"user_districtName"},{"type":"string","name":"user_blockName"},{"type":"string","name":"user_clusterName"},{"type":"string","name":"appName"},{"type":"string","name":"evidences"},{"type":"string","name":"user_stateName"},{"type":"string","name":"domainName"},{"type":"string","name":"domainExternalId"},{"type":"string","name":"childName"},{"type":"string","name":"childType"},{"type":"string","name":"childExternalid"},{"type":"string","name":"level"},{"type":"string","name":"criteriaDescription"},{"type":"string","name":"programDescription"},{"type":"string","name":"solutionDescription"},{"type":"string","name":"label"},{"type":"string","name":"imp_project_id"},{"type":"string","name":"imp_project_title"},{"type":"string","name":"imp_project_goal"},{"type":"string","name":"imp_project_externalId"},{"type":"string","name":"ancestorName"},{"type":"string","name":"scoringSystem"},{"type":"string","name":"domainLevel"},{"type":"string","name":"domainScore"},{"name":"criteriaLevelReport","type":"boolean"},{"type":"string","name":"user_schoolName"},{"type":"string","name":"user_schoolId"},{"type":"string","name":"user_schoolUDISE_code"},{"type":"string","name":"solution_type"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_boardName"},{"type":"string","name":"district_externalId"},{"type":"string","name":"state_externalId"},{"type":"string","name":"block_externalId"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"user_type"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}' + +survey_query_spec = '{"queryType":"scan","dataSource":"sl-survey","resultFormat":"list","columns":["completedDate","createdAt","createdBy","criteriaExternalId","criteriaId","criteriaName","surveyId","surveyName","surveySubmissionId","questionAnswer","questionECM","questionExternalId","questionId","questionName","questionResponseLabel","questionResponseType","solutionExternalId","solutionId","solutionName","updatedAt","instanceParentId","instanceId","instanceParentResponsetype","instanceParentQuestion","questionSequenceByEcm","maxScore","minScore","percentageScore","pointsBasedScoreInParent","totalScore","scoreAchieved","totalpercentage","instanceParentExternalId","instanceParentEcmSequence","remarks","total_evidences","evidence_count","instanceParentCriteriaId","instanceParentCriteriaExternalId","instanceParentCriteriaName","isAPrivateProgram","programId","programName","programExternalId","questionResponseLabel_number","channel","parent_channel","appName","organisation_name","user_subtype","user_type","board_name","district_code","district_name","district_externalId","block_code","block_name","block_externalId","school_code","school_name","school_externalId","cluster_code","cluster_name","cluster_externalId","state_code","state_name","state_externalId","organisation_id","evidences"],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]}' + +survey_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-survey","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"surveyId"},{"type":"string","name":"surveyName"},{"type":"string","name":"surveySubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"evidences"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"appName"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_subtype"},{"type":"string","name":"user_type"},{"type":"string","name":"board_name"},{"type":"string","name":"district_code"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_code"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"school_code"},{"type":"string","name":"school_name"},{"type":"string","name":"school_externalId"},{"type":"string","name":"cluster_code"},{"type":"string","name":"cluster_name"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"state_code"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}' + +intervals = '["1901-01-01T00:00:00+00:00/2020-01-01T00:00:00+00:00","2020-01-01T00:00:00+00:00/2020-06-01T00:00:00+00:00","2020-06-01T00:00:00+00:00/2021-01-01T00:00:00+00:00","2021-01-01T00:00:00+00:00/2021-06-01T00:00:00+00:00","2021-06-01T00:00:00+00:00/2022-01-01T00:00:00+00:00","2022-01-01T00:00:00+00:00/2022-03-01T00:00:00+00:00","2022-03-01T00:00:00+00:00/2022-06-01T00:00:00+00:00","2022-06-01T00:00:00+00:00/2022-09-01T00:00:00+00:00","2022-09-01T00:00:00+00:00/2023-01-01T00:00:00+00:00"]' + + +[KAFKA] + +url = {{ml_kafka_host | default(groups['processing-cluster-kafka'][0]+':9092')}} + +observation_raw_topic = {{ env_name }}.ml.observation.raw + +observation_druid_topic = {{ env_name }}.ml.observation.druid + +observation_evidence_druid_topic = {{ env_name }}.ml.observation.evidence.druid + +survey_evidence_druid_topic = {{ env_name }}.ml.survey.evidence.druid + +survey_raw_topic = {{ env_name }}.ml.survey.raw + +survey_druid_topic = {{ env_name }}.ml.survey.druid + +survey_meta_druid_topic = {{ env_name }}.ml.survey.meta + +survey_completed_druid_topic = {{ env_name }}.ml.survey.status.completed + +survey_started_druid_topic = {{ env_name }}.ml.survey.status.started + +survey_inprogress_druid_topic = {{ env_name }}.ml.survey.status.inprogress + +observation_meta_druid_topic = {{ env_name }}.ml.observation.meta + +observation_started_druid_topic = {{ env_name }}.ml.observation.status.started + +observation_inprogress_druid_topic = {{ env_name }}.ml.observation.status.inprogress + +observation_completed_druid_topic = {{ env_name }}.ml.observation.status.completed + +[LOGS] + +observation_streaming_success_error = /opt/sparkjobs/ml-analytics-service/logs/observation/ + +observation_streaming_evidence_success_error = /opt/sparkjobs/ml-analytics-service/logs/observation/evidence/ + +observation_status_success_error = /opt/sparkjobs/ml-analytics-service/logs/observation/status/ + +project_success_error = /opt/sparkjobs/ml-analytics-service/logs/project/ + +survey_evidence_streaming_success_error = /opt/sparkjobs/ml-analytics-service/logs/project/evidence/ + +survey_streaming_success_error = /opt/sparkjobs/ml-analytics-service/logs/survey/ + +survey_status_success_error = /opt/sparkjobs/ml-analytics-service/logs/survey/status/ + + +[CLOUD] + +container_name = {{ cloud_storage_telemetry_bucketname }} + +[OUTPUT_DIR] + +project = /opt/sparkjobs/source/projects/output + +observation_status = /opt/sparkjobs/source/observations/status/output + +observation_distinctCount_status = /opt/sparkjobs/source/observations/distinctCount/output + +observation_distinctCount_domain = /opt/sparkjobs/source/observations/distinctCount_domain/output + +observation_distinctCount_domain_criteria = /opt/sparkjobs/source/observations/distinctCount_domain_criteria/output + +projects_distinctCount = /opt/sparkjobs/source/projects/distinctCount/output + +projects_distinctCount_prgmlevel = /opt/sparkjobs/source/projects/distinctCountPrglevel/output + +survey_distinctCount_status = /opt/sparkjobs/source/survey/distinctCount/output + +project_rollup = /opt/sparkjobs/source/projects/output_rollup + +observation_status_rollup = /opt/sparkjobs/source/observations/output_rollup + +survey_rollup = /opt/sparkjobs/source/survey/output + +survey_status = /opt/sparkjobs/source/survey/status/output + +observation_sub_ids = /opt/sparkjobs/ml-analytics-service/observations/submissions.csv + +observation_druid_data = /opt/sparkjobs/source/observations/ + +survey_sub_ids = /opt/sparkjobs/ml-analytics-service/survey/submissions.csv + +survey_druid_data = /opt/sparkjobs/source/survey/ + +program_text_file = /opt/sparkjobs/ml-analytics-service/projects/program_ids.txt + +[SLACK] + +token = {{ ml_analytics_slack_token | default('') }} + +channel = {{ ml_analytics_slack_channel | default('') }} + + +[VAM] + +druid_query_url = {{ ml_druid_query | default('') }} + +program_dashboard_data = {{ ml_program_data | default('') }} + + +[COMMON] + +cloud_module_path = /opt/sparkjobs/ml-analytics-service/cloud_storage + +observation_blob_path = observation/status/ + +projects_blob_path = projects/ + +observation_distinctCount_blob_path = observation/distinctCount/ + +observation_distinctCount_domain_blob_path = observation/distinctCount_domain/ + +observation_distinctCount_domain_criteria_blob_path = observation/distinctCount_domain_criteria/ + +projects_distinctCnt_blob_path = projects/distinctCount/ + +projects_distinctCnt_prgmlevel_blob_path = projects/distinctCountPrglevel/ + +survey_distinctCount_blob_path = survey/distinctCount/ + +projects_rollup_blob_path = projects/rollup + +observation_rollup_blob_path = observation/rollup + +survey_rollup_blob_path = survey/rollup/ + +survey_blob_path = survey/status/ + +projects_program_csv = {{ ml_program_blob_path | default('') }} + +observation_batch_ingestion_data_del = observation/batchDeletion + +survey_batch_ingestion_data_del = survey/batchDeletion + +cname_url = {{ cloud_storage_url }}/{{ cloud_storage_samiksha_bucketname }} + +nvsk_imp_projects_data_local_path = /opt/sparkjobs/ml-analytics-service/urgent_data_metrics/output/ + +nvsk_imp_projects_data_blob_path = Manage_Learn_Data/micro_improvement/ + +[API_CREDENTIALS] + +client_id = {{ ml_analytics_client_id }} + +client_secret = {{ ml_analytics_client_secret }} + +username = {{ ml_analytics_username }} + +password = {{ ml_analytics_password }} + +[JSON_VARIABLE] + +createdBy = {{ ml_analytics_createdBy }} + +container = {{ ml_analytics_reports_container }} + +store = {{ ml_analytics_reports_store }} + +[ML_CORE_SERVICE_URL] + +url = http://{{private_ingressgateway_ip}}/ml-core + +[SERVICES] + +faust_applications_list = {{ faust_applications_list }} diff --git a/ansible/roles/stack-sunbird/templates/ml-core-service.env b/ansible/roles/stack-sunbird/templates/ml-core-service.env index 56313ff44f..463c010d1b 100755 --- a/ansible/roles/stack-sunbird/templates/ml-core-service.env +++ b/ansible/roles/stack-sunbird/templates/ml-core-service.env @@ -69,4 +69,12 @@ KAFKA_GROUP_ID={{ml_core_kafka_group_id | default("ml-core-service")}} PROGRAM_USERS_JOINED_TOPIC={{ml_core_program_users_joined_topic | default (env_name+".programuser.info")}} #TimeOffset with respect to UTC -TIMEZONE_DIFFRENECE_BETWEEN_LOCAL_TIME_AND_UTC={{ml_core_timezone_diffrenece_between_local_time_and_utc | default ("+05:30")}} \ No newline at end of file +TIMEZONE_DIFFRENECE_BETWEEN_LOCAL_TIME_AND_UTC={{ml_core_timezone_diffrenece_between_local_time_and_utc | default ("+05:30")}} + + +#Validate Entities enable/Disable +VALIDATE_ENTITIES={{ml_service_validate_entities | default ("ON")}} + +#Public base url of sunbird +PUBLIC_BASE_URL={{ml_core_public_base_url | default("")}} + diff --git a/ansible/roles/stack-sunbird/templates/ml-survey-service.env b/ansible/roles/stack-sunbird/templates/ml-survey-service.env index ccda45606e..51f5828455 100644 --- a/ansible/roles/stack-sunbird/templates/ml-survey-service.env +++ b/ansible/roles/stack-sunbird/templates/ml-survey-service.env @@ -65,3 +65,9 @@ KEYCLOAK_PUBLIC_KEY_PATH={{ml_survey_keycloak_public_key_path|default("keycloak- ## portal url of env APP_PORTAL_BASE_URL={{ proto }}://{{ domain_name }} + +#Program join enable/disable +PROGRAM_JOIN_ON_OFF={{ml_program_join_on_off | default ("ON")}} + +#validate entities enable/disable +VALIDATE_ENTITIES={{ml_service_validate_entities | default ("ON")}} diff --git a/ansible/roles/stack-sunbird/templates/sunbird_report-service.env b/ansible/roles/stack-sunbird/templates/sunbird_report-service.env index a2ebfce02a..b13358fcac 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_report-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_report-service.env @@ -29,3 +29,13 @@ sunbird_cache_ttl={{sunbird_cache_ttl}} sunbird_report_sas_expiry_in_minutes=3600 sunbird_dataservice_url={{sunbird_dataservice_url}} +# 6.0.0 + +sunbird_cloud_storage_provider={{cloud_service_provider}} +cloud_private_storage_accountname={{cloud_private_storage_accountname | default("")}} +cloud_private_storage_secret={{cloud_private_storage_secret | default("")}} +cloud_storage_privatereports_bucketname={{cloud_storage_privatereports_bucketname | default("reports")}} +cloud_storage_resourceBundle_bucketname={{cloud_storage_label_bucketname | default("label")}} +cloud_private_storage_region={{cloud_private_storage_region | default("ap-south-1")}} +cloud_private_storage_project={{cloud_private_storage_project | default("")}} +cloud_private_storage_endpoint={{cloud_private_storage_endpoint | default("")}} diff --git a/deploy/jenkins/jenkins-server-setup.sh b/deploy/jenkins/jenkins-server-setup.sh index d96e3b4228..77d29434a0 100755 --- a/deploy/jenkins/jenkins-server-setup.sh +++ b/deploy/jenkins/jenkins-server-setup.sh @@ -12,10 +12,10 @@ echo -e "\n\e[0;32m${bold}Installating JDK8${normal}\n" apt-get install -y openjdk-8-jdk echo -e "\n\e[0;32m${bold}Installating Jenkins${normal}" -wget -q -O - https://pkg.jenkins.io/debian-stable/jenkins.io.key | apt-key add - +wget -q -O - https://pkg.jenkins.io/debian-stable/jenkins.io-2023.key | apt-key add - apt-add-repository "deb https://pkg.jenkins.io/debian-stable binary/" apt-get update -apt-get install -y jenkins=2.319.3 +apt-get install -y jenkins=2.346.3 echo -e "\n\e[0;32m${bold}Installating PIP${normal}" apt-get install -y python-pip @@ -72,6 +72,14 @@ echo -e "\n\e[0;32m${bold}Installating Ansible${normal}" pip uninstall -y ansible pip3 install ansible==2.8.19 +echo -e "\n\e[0;32m${bold}Installing oci cli ${normal}" +oci_cli_setup_zip="https://github.com/oracle/oci-cli/releases/download/v3.22.0/oci-cli-3.22.0-Ubuntu-18.04-Offline.zip" +wget $oci_cli_setup_zip -O /tmp/ocicli.zip +unzip /tmp/ocicli.zip -d /tmp +cd /tmp +./oci-cli-installation/install.sh --install-dir /var/lib/jenkins --exec-dir /var/lib/jenkins --script-dir /var/lib/jenkins --accept-all-defaults +mv /var/lib/jenkins/oci /usr/bin + echo -e "\n\e[0;32m${bold}Installating azure cli${normal}" apt-get install ca-certificates curl apt-transport-https lsb-release gnupg curl -sL https://packages.microsoft.com/keys/microsoft.asc | @@ -107,6 +115,10 @@ echo -e "\n\e[0;32m${bold}Creating bashrc for jenkins user ${normal}" cp /etc/skel/.bashrc /var/lib/jenkins chown jenkins:jenkins /var/lib/jenkins/.bashrc +echo -e "\n\e[0;32m${bold}Creating profile for jenkins user ${normal}" +echo "export OCI_CLI_AUTH=instance_principal" > /var/lib/jenkins/.profile +chown jenkins:jenkins /var/lib/jenkins/.profile + echo -e "\n\e[0;32m${bold}Setting timezone to IST ${normal}" timedatectl set-timezone Asia/Kolkata @@ -115,6 +127,7 @@ su jenkins bash -c "curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34. echo -e "\n\e[0;32m${bold}Installing jmespath${normal}" sudo apt install -y python3-jmespath +sudo apt install python-jmespath #python2 #### Kubernetes Tools #### @@ -155,7 +168,7 @@ rm openjdk-11.0.2_linux-x64_bin.tar.gz #Install maven 3.6.3 echo -e "\n\e[0;32m${bold}Installating maven 3.6.3${normal}" -wget https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz +wget https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz tar -xf apache-maven-3.6.3-bin.tar.gz mv apache-maven-3.6.3 /opt/ mv /opt/apache-maven-3.6.3/bin/mvn /opt/apache-maven-3.6.3/bin/mvn3.6 diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml new file mode 100644 index 0000000000..28b220559d --- /dev/null +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml @@ -0,0 +1,108 @@ + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + 10 + -1 + 1 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + Build/KnowledgePlatform/InquiryFlinkJob + false + + + image_tag + <font color=darkgreen size=2><b>OPTIONAL: Specify the tag to upload a specific image version to the container registry.</b></font> + + false + + + artifact_source + <font color=dimgray size=2><b> +ArtifactRepo - Push the docker image to container registry. +</b></font> + + + ArtifactRepo + + + + + + + 0 + 0 + + false + project + false + + + + + + + + Build/KnowledgePlatform/InquiryFlinkJob + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/Sunbird-inQuiry/data-pipeline.git + + + + + ${inquiry_pipeline_branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + kubernetes/pipelines/upload/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml index 5bc1fe4d34..c6304ef3b4 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml @@ -35,6 +35,30 @@ refs/heads/${public_repo_branch} true + + CLOUD_STORE_GROUP_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk group id</li> +</b></font> + org.sunbird + false + + + CLOUD_STORE_ARTIFACT_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk artifact id</li> +</b></font> + cloud-store-sdk_2.12 + false + + + CLOUD_STORE_VERSION + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk version</li> +</b></font> + 1.4.0 + false + @@ -48,12 +72,7 @@ - - - H/15 * * * * - false - - + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml index d2c038b5fa..b4deb5e46b 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml @@ -1,6 +1,6 @@ - + hudson.model.ParametersDefinitionProperty com.sonyericsson.rebuild.RebuildSettings @@ -19,26 +19,47 @@ - + false false - github_release_tag + inquiry_release_tag <font style="color:dimgray;font-size:14px;"><b> <li>To build from a tag, use refs/tags/github_tag</li> <li>To build from a branch, use refs/heads/github_branch</li> -<li>The default value of ${public_repo_branch} will be the release / tag version set in global configuration</li> -<li>To build from a differnt branch, replace the ${public_repo_branch} with your branch</li> +<li>The default value of ${inquiry_service_build_branch_or_tag} will be the release / tag version set in global configuration</li> +<li>To build from a differnt branch, replace the ${inquiry_service_build_branch_or_tag} with your branch</li> </b></font> - refs/heads/${public_repo_branch} + refs/heads/${inquiry_service_build_branch_or_tag} true + + core_release_tag + <font style="color:dimgray;font-size:14px;"><b> +<li>Provide this value to build core components from Knowlg BB</li> +<li>To build from a tag, use refs/tags/github_tag</li> +<li>To build from a branch, use refs/heads/github_branch</li> +<li>The default value of ${inquiry_core_build_branch_or_tag} will be the release / tag version set in global configuration</li> +<li>To build from a differnt branch, replace the ${inquiry_core_build_branch_or_tag} with your branch</li> +</b></font> + refs/heads/${inquiry_core_build_branch_or_tag} + false + + + core_repo_link + Knowlg Core Repo + + + https://github.com/project-sunbird/knowledge-platform.git + + + - + 0 0 @@ -49,31 +70,29 @@ - - - H/15 * * * * - false - - + - - + + 2 - https://github.com/project-sunbird/knowledge-platform.git + https://github.com/Sunbird-inQuiry/inquiry-api-service.git - ${github_release_tag} + ${inquiry_release_tag} + false + + build/assessment-service/Jenkinsfile false false - \ No newline at end of file + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml index 7b47e1972e..f3f5d6477c 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + CLOUD_STORE_GROUP_ID + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + CLOUD_STORE_ARTIFACT_ID + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12 + ${cloud_store_artifact_id} + false + + + CLOUD_STORE_VERSION + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml index 2cdac260ad..a12dd235f1 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + cloud_store_group_id + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + cloud_store_artifact_id + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk + ${cloud_store_artifact_id} + false + + + cloud_store_version + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + @@ -51,7 +69,7 @@ - H/15 * * * * + false diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml index 89116c8406..4eecb95a61 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + cloud_store_group_id + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + cloud_store_artifact_id + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk + ${cloud_store_artifact_id} + false + + + cloud_store_version + Set the Cloud store sdk version. e.g 1.4.6 + ${cloud_store_version} + false + diff --git a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/AnalyticsCore/config.xml b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/AnalyticsCore/config.xml index 2b9e72c45c..5c202a759c 100644 --- a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/AnalyticsCore/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/AnalyticsCore/config.xml @@ -36,6 +36,30 @@ refs/heads/${public_repo_branch} true + + CLOUD_STORE_GROUP_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk group id</li> +</b></font> + org.sunbird + false + + + CLOUD_STORE_ARTIFACT_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk artifact id</li> +</b></font> + cloud-store-sdk_2.12 + false + + + CLOUD_STORE_VERSION + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk version</li> +</b></font> + 1.4.0 + false + @@ -51,7 +75,7 @@ - H/15 * * * * + false diff --git a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/CoreDataProducts/config.xml b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/CoreDataProducts/config.xml index a458418b24..8ff4dc5426 100644 --- a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/CoreDataProducts/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/CoreDataProducts/config.xml @@ -27,6 +27,27 @@ github_release_tag + <font color=red size=2><b>CAUTION: If the value is blank, latest code will be built. Specify github tag name to build from a tag.</b></font> + false + + + CLOUD_STORE_GROUP_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk group id</li> +</b></font> + org.sunbird + false + + + CLOUD_STORE_ARTIFACT_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk artifact id</li> +</b></font> + cloud-store-sdk_2.12 + false + + + CLOUD_STORE_VERSION <font style="color:dimgray;font-size:14px;"><b> <li>To build from a tag, use refs/tags/github_tag</li> <li>To build from a branch, use refs/heads/github_branch</li> @@ -51,7 +72,7 @@ - H/15 * * * * + false diff --git a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml index 11f30a0d33..62a0cd1f68 100644 --- a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + CLOUD_STORE_GROUP_ID + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + CLOUD_STORE_ARTIFACT_ID + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12 + ${cloud_store_artifact_id} + false + + + CLOUD_STORE_VERSION + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + diff --git a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml new file mode 100644 index 0000000000..93b0eba8f7 --- /dev/null +++ b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml @@ -0,0 +1,77 @@ + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + 10 + -1 + 1 + + + + + false + false + + + + + inquiry_release_tag + <font style="color:dimgray;font-size:14px;"><b> +<li>To build from a tag, use refs/tags/github_tag</li> +<li>To build from a branch, use refs/heads/github_branch</li> +<li>The default value of ${inquiry_pipeline_branch_or_tag} will be the release / tag version set in global configuration</li> +<li>To build from a differnt branch, replace the ${inquiry_pipeline_branch_or_tag} with your branch</li> +</b></font> + refs/heads/${inquiry_pipeline_branch_or_tag} + true + + + + + 0 + 0 + + false + project + false + + + + + + + + + + 2 + + + https://github.com/Sunbird-inQuiry/data-pipeline.git + + + + + ${inquiry_release_tag} + + + false + + + + kubernetes/pipelines/build/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml index 088b9fa27f..9b997d2ce1 100644 --- a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + cloud_store_group_id + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + cloud_store_artifact_id + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12 + ${cloud_store_artifact_id} + false + + + cloud_store_version + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + @@ -49,12 +67,7 @@ - - - H/15 * * * * - false - - + diff --git a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml index df82bece0b..f8041941e2 100644 --- a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + cloud_store_group_id + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + cloud_store_artifact_id + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12 + ${cloud_store_artifact_id} + false + + + cloud_store_version + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml new file mode 100644 index 0000000000..af888dbe75 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml @@ -0,0 +1,149 @@ + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + 10 + -1 + 2 + + + + + false + false + + + + + private_branch + + choice-parameter-2544395024638227 + 1 + + true + + + + true + + + InquiryFlinkJob + Deploy/dev/KnowledgePlatform/InquiryFlinkJob + + + ET_FORMATTED_HTML + true + + + inquiry_release_tag + <font style="color:dimgray;font-size:14px;"><b> +<li>To build from a tag, use refs/tags/github_tag</li> +<li>To build from a branch, use refs/heads/github_branch</li> +<li>The default value of ${inquiry_pipeline_branch_or_tag} will be the release / tag version set in global configuration</li> +<li>To build from a differnt branch, replace the ${inquiry_pipeline_branch_or_tag} with your branch</li> +</b></font> + refs/heads/${inquiry_pipeline_branch_or_tag} + false + + + image_tag + <font color=red size=2><b>CAUTION: If the value is blank, image tag will be taken from the latest metadata.json.</b></font> + + false + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + ArtifactUpload/dev/KnowledgePlatform/InquiryFlinkJob + false + + + job_names_to_deploy + <font color=green size=2><b>Choose the job names to deploy. Multi-selection is available.</b></font> + choice-parameter-1273072434092073 + 1 + + true + + + + InquiryFlinkJob + Deploy/dev/KnowledgePlatform/InquiryFlinkJob + + + PT_MULTI_SELECT + false + 1 + + + + + 0 + 0 + + false + project + false + + + + + + + + + + 2 + + + https://github.com/Sunbird-inQuiry/data-pipeline.git + + + + + ${inquiry_release_tag} + + + false + + + + true + false + + 0 + false + + + + kubernetes/pipelines/deploy/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/jobs/Cassandra/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/jobs/Cassandra/config.xml index 1e9db63132..3304f15fad 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/jobs/Cassandra/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/jobs/Cassandra/config.xml @@ -1,219 +1,261 @@ - - - false - - - - -1 - 10 - -1 - 2 - - - - - false - false - - - - - absolute_job_path - <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> - ArtifactUpload/dev/Core/Cassandra - false - - - artifact_source - <font color=dimgray size=2><b> -ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrifact from Jenkins job.</b></font> - choice-parameter-9600649228560 - 1 - - true - - - - Cassandra - Deploy/dev/Kubernetes/Cassandra - - - PT_SINGLE_SELECT - false - 1 - - - build_number - - choice-parameter-9600651313765 - 1 - - true - - - - Cassandra - Deploy/dev/Kubernetes/Cassandra - - artifact_source - ET_FORMATTED_HTML - true - - - artifact_version - - choice-parameter-9600653373369 - 1 - - true - - - - Cassandra - Deploy/dev/Kubernetes/Cassandra - - artifact_source - ET_FORMATTED_HTML - true - - - private_branch - - choice-parameter-2544395024638227 - 1 - - true - - - - true - - - Cassandra - Deploy/dev/Kubernetes/Cassandra - - - ET_FORMATTED_HTML - true - - - branch_or_tag - - choice-parameter-2620434998790477 - 1 - + true + + + + Cassandra + Deploy/dev/Kubernetes/Cassandra + + + PT_SINGLE_SELECT + false + 1 + + + build_number + choice-parameter-591902397549104 + 1 + + true + + + + Cassandra + Deploy/dev/Kubernetes/Cassandra + + artifact_source + ET_FORMATTED_HTML + true + + + artifact_version + choice-parameter-591902398907654 + 1 + + true + + + + Cassandra + Deploy/dev/Kubernetes/Cassandra + + artifact_source + ET_FORMATTED_HTML + false + + + private_branch + choice-parameter-2544395024638227 + 1 + + true + + + + true + + + Cassandra + Deploy/dev/Kubernetes/Cassandra + + + ET_FORMATTED_HTML + true + + + branch_or_tag + choice-parameter-2620434998790477 + 1 + - true - - - - true - - - Cassandra - Deploy/dev/Kubernetes/Cassandra - - - ET_FORMATTED_HTML - true - - - - - 0 - 0 - - false - project - false - - - - - - - - ArtifactUpload/dev/Core/Cassandra - - SUCCESS - 0 - BLUE - true - - - - - - - - 2 - - - https://github.com/project-sunbird/sunbird-devops.git - - - - - ${branch_or_tag} - - - false - - - - true - false - - 0 - false - - - - pipelines/deploy/cassandra-deploy/Jenkinsfile - false - - - false + } + else + return """<b>This parameter is not used</b>""" + true + + + + true + + + Cassandra + Deploy/dev/Kubernetes/Cassandra + + + ET_FORMATTED_HTML + true + + + script_repo + Enter the repo url from which keyspace script folder to be uploaded + https://github.com/Sunbird-Lern/sunbird-utils.git + false + + + script_repo_branch_or_tag + Enter the branch or tag for keyspace script repo + ${branch_or_tag} + false + + + cassandra_keyspace_to_migrate + choice-parameter-598043621453772 + 1 + + true + + + + Cassandra + Deploy/dev/Kubernetes/Cassandra + + + PT_SINGLE_SELECT + false + 1 + + + + + 0 + 0 + + false + project + false + + + + + + + + ArtifactUpload/dev/Core/Cassandra + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${public_repo_branch} + + + false + + + + true + false + + 0 + false + + + + pipelines/deploy/cassandra-deploy/Jenkinsfile + false + + + false \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/jobs/InquiryUploadSchema/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/jobs/InquiryUploadSchema/config.xml new file mode 100644 index 0000000000..1735ad1e56 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/jobs/InquiryUploadSchema/config.xml @@ -0,0 +1,139 @@ + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + + -1 + -1 + -1 + 5 + + + + false + + + false + false + + + + + private_branch + choice-parameter-2544395024638227 + 1 + + true + + + + true + + + InquiryUploadSchema + Deploy/dev/Kubernetes/InquiryUploadSchema + + + ET_FORMATTED_HTML + true + + + branch_or_tag + choice-parameter-2620434998790477 + 1 + + true + + + + true + + + InquiryUploadSchema + Deploy/dev/Kubernetes/InquiryUploadSchema + + + ET_FORMATTED_HTML + true + + + schema_repo + Enter the repo url from which schema folder to be uploaded + https://github.com/Sunbird-inQuiry/inquiry-api-service.git + false + + + schema_repo_branch_or_tag + Enter the branch or tag for schema repo + false + + + + + 0 + 0 + + false + project + false + + + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/upload/schema/inquiry/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/jobs/LernFlinkJobs/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/jobs/LernFlinkJobs/config.xml index 396590af95..aae4f06081 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/jobs/LernFlinkJobs/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/jobs/LernFlinkJobs/config.xml @@ -106,7 +106,8 @@ return """<b>This parameter is not used</b>""" 'assessment-aggregator', 'enrolment-reconciliation', 'notification-job', -'program-user-info'] +'program-user-info', +'legacy-certificate-migrator'] true diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml index 859b804b88..f669caa862 100644 --- a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml @@ -26,7 +26,7 @@ @@ -114,4 +114,4 @@ return """<b>This parameter is not used</b>""" false - \ No newline at end of file + diff --git a/deploy/oci-provisioning.yaml b/deploy/oci-provisioning.yaml new file mode 100644 index 0000000000..2c8f378350 --- /dev/null +++ b/deploy/oci-provisioning.yaml @@ -0,0 +1,816 @@ +# Notes: +# DISCLAIMER: This is a basic development setup. This should not be used for production setup. +# How To Run: +# Open a cloud shell session in your home region +# Set the env variables +# export COMPARTMENT_OCID='ocid1.compartment.oc1..aaaaaaaa' ## compartment id for creating resources +# export GEO_IP_LIC_KEY=xyxkjlas ## lic key for downloading geo ip +# export OSS_NAMESPACE=ljlajsdflj ### oci tenancy namespace +# export INSTANCE_PUBLIC_KEY='ssh-rsa AAAAB' ## ssh public key for logging in to compute instances + +# git clone https://github.com/project-sunbird/sunbird-devops.git +# cd sunbird-devops && git checkout tags/release-6.0.0 -b release-6.0.0 +# cd deploy/ +# ansible-playbook oci-provisioning.yaml + +# Prerequisites: +# Login to oci console with admin user + +############### Maxmind licence key ########################### +# To create license key +# Goto: https://www.maxmind.com/en/accounts/current/license-key +# Click: Generate a new license key. +# ############################################################## + +--- +- name: Create Infrastructure + collections: + - oracle.oci + hosts: localhost + vars: + vcn_cidr_block: "{{ lookup('env', 'VCN_CIDR', default='10.90.0.0/16') }}" + public_subnet_cidr_block: "{{ lookup('env', 'PUBLIC_SUBNET_CIDR', default='10.90.1.0/24') }}" + private_subnet_cidr_block: "{{ lookup('env', 'PRIVATE_SUBNET_CIDR', default='10.90.2.0/24') }}" + vcn_name: "SunbirdEd-VCN" + vcn_dns_label: "sunbirded" + compartment_id: "{{ lookup('env', 'COMPARTMENT_OCID') }}" + tenancy_id: "{{ lookup('env', 'OCI_TENANCY')}}" + region: "{{ lookup('env', 'OCI_REGION')}}" + namespace_name: "{{ lookup('env', 'OSS_NAMESPACE') }}" + geoip_licence_key: "{{ lookup('env', 'GEO_IP_LIC_KEY') }}" + authorized_key: "{{ lookup('env', 'INSTANCE_PUBLIC_KEY') }}" + quad_zero_route: "0.0.0.0/0" + instance_shape: "VM.Standard.E3.Flex" + intance_os_version: "{{ lookup('env', 'UBUNTU_VERSION', default='18.04') }}" + k8s_version: "{{ lookup('env', 'K8S_VERSION', default='v1.27.2') }}" + svc_user_name: "{{ lookup('env', 'SB_SVC_USER', default='sunbirded-svc-usr') }}" + svc_user_group: "{{ lookup('env', 'SB_SVC_GROUP', default='sunbirded-svc-grp') }}" + dynamic_group_name: "{{ lookup('env', 'SB_DG_GROUP', default='sunbirded-dg') }}" + env_name: "{{ lookup('env', 'SB_ENV_NAME', default='dev') }}" + ocpu: 1 + memory_in_gbs: 16 + oke_ocpu: 1 + oke_memory_in_gbs: 16 + private_vm_hdd_in_gb: 60 + public_vm_hdd_in_gb: 250 + private_vm_names: + - kp + - dp + - learning + - db + - druid + public_vm_names: + - jenkins + private_containers: + - private + - artifacts + - management + - e-credentials + - contents + - privatereports + - telemetry-data-store + - data-store + - manage-learn + - uci + - dial + - label + - flink + - certqr + - chatbot + - playercdn + - schema + - sourcing + - offlineinstaller + - termsandcondition + - dev-user + - desktopappcrashlogs + public_containers: + - public + - publicreports + repo_name: "{{ lookup('env', 'SB_REPO', default='sunbirded-repo') }}" + repo_list: + - adminutil + - assessment-service + - bot + - cert_registry_service + - cert_service + - content-service + - creation-player + - data-pipeline + - discussion-middleware + - echo-server + - enc_service + - gotenberg + - groups_service + - hawkeye + - knowledge-mw-service + - knowledge-platform-jobs + - kong + - learner_service + - lms_service + - ml-core-service + - ml-projects-service + - ml-reports-service + - ml-survey-service + - nodebb + - notification_service + - odk + - opensaber + - player + - print-service + - program-service + - proxy + - report-service + - router + - search-service + - secor + - sunbird-analytics-service + - sunbird-datapipeline + - sunbird-dial-service + - sunbird-rc-certificate-api + - sunbird-rc-certificate-signer + - sunbird-rc-core + - taxonomy-service + - telemetry-service + - uci-inbound + - uci-orchestrator + - uci-outbound + - uci-transformer + - uci_api + + tasks: + - name: Check pre-requisites + fail: + msg: "Environment variable {{item}} not set. Please declare an environment variable with an appropriate value for the sample to work." + when: item not in ansible_env + with_items: + - "COMPARTMENT_OCID" + - "OSS_NAMESPACE" + - "GEO_IP_LIC_KEY" + - "INSTANCE_PUBLIC_KEY" + + - name: Create a VCN + oci_network_vcn: + compartment_id: "{{ compartment_id }}" + display_name: "{{ vcn_name }}" + cidr_block: "{{ vcn_cidr_block }}" + dns_label: "{{ vcn_dns_label }}" + register: vcn + tags: + - network + + - set_fact: + vcn_id: "{{ vcn.vcn.id }}" + cacheable: yes + tags: + - network + + - name: Get availability domains for vlans + oci_identity_availability_domain_facts: + compartment_id: "{{compartment_id}}" + register: oci_ads + tags: + - always + + - name: Get all services + oci_network_service_facts: + register: oci_services + tags: + - always + + - name: Get all images + oci_compute_image_facts: + compartment_id: "{{compartment_id}}" + shape: "{{instance_shape}}" + operating_system : "Canonical Ubuntu" + operating_system_version: "{{intance_os_version}}" + register: oci_images + tags: + - always + + - name: Get OKE images + oci_compute_image_facts: + compartment_id: "{{compartment_id}}" + shape: "{{instance_shape}}" + operating_system : "Oracle Linux" + operating_system_version: "7.9" + register: oke_node_images + tags: + - always + + - name: Get Region + shell: oci iam region list --query "data[?name=='{{region}}'].key | [0]" --raw-output | tr '[:upper:]' '[:lower:]' + register: region_code + tags: + - always + + - set_fact: + availability_domain: "{{oci_ads.availability_domains[0].name}}" + oci_all_service_id: "{{oci_services.services[1].id}}" + oci_all_service_cidr: "{{oci_services.services[1].cidr_block}}" + ocir_region: "{{region_code.stdout}}" + instance_image_id: "{{oci_images.images[0].id}}" + oke_image_id: "{{oke_node_images.images[0].id}}" + node_shape: "{{ instance_shape }}" + node_pool_source_details: + source_type: "IMAGE" + image_id: "{{oke_node_images.images[0].id}}" + tags: + - always + + - name: templating bootstrap + template: + src: "{{ item }}.j2" + dest: "/tmp/{{item}}.sh" + with_items: + - oci_vms_bootstrap + tags: + - always + + - name: Create a new Internet Gateway + oci_network_internet_gateway: + compartment_id: "{{ compartment_id }}" + vcn_id: "{{ vcn_id }}" + name: "IG" + is_enabled: 'yes' + state: 'present' + register: igw + tags: + - network + + - set_fact: + ig_id: "{{ igw.internet_gateway.id }}" + public_route_table_rules: + - cidr_block: "{{ quad_zero_route }}" + network_entity_id: "{{ igw.internet_gateway.id }}" + tags: + - network + + - name: Create a new NAT Gateway + oci_network_nat_gateway: + compartment_id: "{{ compartment_id }}" + vcn_id: "{{ vcn_id }}" + name: "NG" + state: 'present' + register: ngw + tags: + - network + + - name: Create a new Service Gateway + oci_network_service_gateway: + compartment_id: "{{ compartment_id }}" + vcn_id: "{{ vcn_id }}" + services: + - service_id: "{{oci_all_service_id}}" + display_name: "SVC_GW" + state: 'present' + register: sgw + tags: + - network + + - set_fact: + ng_id: "{{ ngw.nat_gateway.id }}" + sg_id: "{{ sgw.service_gateway.id }}" + private_route_table_rules: + - destination: "{{ oci_all_service_cidr }}" + destination_type: SERVICE_CIDR_BLOCK + network_entity_id: "{{ sgw.service_gateway.id }}" + - destination: "{{ quad_zero_route }}" + destination_type: CIDR_BLOCK + network_entity_id: "{{ ngw.nat_gateway.id }}" + tags: + - network + + - name: Create route table to connect internet gateway to the VCN + oci_network_route_table: + compartment_id: "{{ compartment_id }}" + vcn_id: "{{ vcn_id }}" + name: "PUB-SL-RT" + route_rules: "{{ public_route_table_rules }}" + state: 'present' + register: public_rt + tags: + - network + + - set_fact: + public_rt_id: "{{ public_rt.route_table.id }}" + tags: + - network + + - name: Create route table to connect Nat gateway to the VCN + oci_network_route_table: + compartment_id: "{{ compartment_id }}" + vcn_id: "{{ vcn_id }}" + name: "PVT-SL-RT" + route_rules: "{{ private_route_table_rules }}" + state: 'present' + register: private_rt + tags: + - network + + - set_fact: + private_rt_id: "{{ private_rt.route_table.id }}" + tags: + - network + + - name: Create security list for Public Subnet + oci_network_security_list: + display_name: PUB-SL + compartment_id: "{{ compartment_id }}" + vcn_id: "{{ vcn_id }}" + egress_security_rules: + - description: "Allow outgoing traffic" + destination: "{{ quad_zero_route }}" + protocol: "all" + destination_type: CIDR_BLOCK + is_stateless: false + ingress_security_rules: + - description: "allow all traffic" + protocol: all + source: "{{quad_zero_route}}" + is_stateless: false + source_type: CIDR_BLOCK + register: security_list + tags: + - network + - set_fact: + security_list_id: "{{ security_list.security_list.id }}" + cacheable: yes + tags: + - network + + - name: Create security list for Private Subnet + oci_network_security_list: + display_name: PVT-SL + compartment_id: "{{ compartment_id }}" + vcn_id: "{{ vcn_id }}" + egress_security_rules: + - description: "Allow outgoing traffic" + destination: "{{ quad_zero_route }}" + protocol: "all" + destination_type: CIDR_BLOCK + is_stateless: false + ingress_security_rules: + - description: "allow all traffic" + protocol: all + source: "{{vcn_cidr_block}}" + is_stateless: false + source_type: CIDR_BLOCK + register: pvt_security_list + tags: + - network + - set_fact: + pvt_security_list_id: "{{ pvt_security_list.security_list.id }}" + cacheable: yes + tags: + - network + + - name: Create a Public Subnet + oci_network_subnet: + cidr_block: "{{ public_subnet_cidr_block }}" + compartment_id: "{{ compartment_id }}" + display_name: "PUB-SN" + route_table_id: "{{ public_rt_id }}" + security_list_ids: + - "{{security_list_id}}" + dns_label: "pubsn" + prohibit_public_ip_on_vnic: no + vcn_id: "{{ vcn_id }}" + register: pub_subnet + tags: + - network + - set_fact: + public_subnet_id: "{{ pub_subnet.subnet.id }}" + cacheable: yes + + - name: Create a Private Subnet + oci_network_subnet: + cidr_block: "{{ private_subnet_cidr_block }}" + compartment_id: "{{ compartment_id }}" + display_name: "PVT-SN" + route_table_id: "{{ private_rt_id }}" + security_list_ids: + - "{{pvt_security_list_id}}" + dns_label: "pvtsn" + prohibit_public_ip_on_vnic: yes + vcn_id: "{{ vcn_id }}" + register: pvt_subnet + tags: + - network + - set_fact: + private_subnet_id: "{{ pvt_subnet.subnet.id }}" + cacheable: yes + + - name: Create Private Buckets + oci_object_storage_bucket: + namespace_name: "{{ namespace_name }}" + compartment_id: "{{ compartment_id }}" + public_access_type: NoPublicAccess + storage_tier: Standard + name: "{{env_name}}-{{ item }}" + with_items: "{{ private_containers }}" + tags: + - oss + + - name: Create Public Buckets + oci_object_storage_bucket: + # required + namespace_name: "{{ namespace_name }}" + compartment_id: "{{ compartment_id }}" + public_access_type: ObjectRead + storage_tier: Standard + name: "{{env_name}}-{{ item }}" + with_items: "{{ public_containers }}" + tags: + - oss + + - name: Create a temporary sample file to upload + tempfile: + register: tmpfile1 + tags: + - oss + - name: Put content in the temporary sample file + copy: + content: "This is content for the sample file." + dest: "{{ tmpfile1.path }}" + tags: + - oss + - name: Creating folder structurue + oci_object_storage_object: + namespace_name: "{{ namespace_name }}" + bucket_name: "{{env_name}}-contents" + object_name: "{{ item }}/tmpfile1.txt" + src: "{{ tmpfile1.path }}" + with_items: + - collection-editor + - generic-editor + - content-edition + - v3/preview + - terms-and-conditions + tags: + - oss + + - name: Downloading terms-and-conditions + get_url: + url: https://sunbirdpublic.blob.core.windows.net/installation/terms-and-conditions/terms-and-conditions-v9.html + dest: /tmp/terms-and-conditions-v9.html + mode: '0440' + tags: + - oss + + - name: Uploading T&C + oci_object_storage_object: + namespace_name: "{{namespace_name}}" + bucket_name: "{{env_name}}-contents" + object_name: "terms-and-conditions/terms-and-conditions-v9.html" + src: /tmp/terms-and-conditions-v9.html + content_type: 'text/plain' + tags: + - oss + + - name: Downloading neo4j + get_url: + url: http://dist.neo4j.org/neo4j-community-3.3.9-unix.tar.gz + dest: /tmp/neo4j-community-3.3.9-unix.tar.gz + mode: '0440' + tags: + - oss + + - name: Uploading Neo4j + oci_object_storage_object: + namespace_name: "{{namespace_name}}" + bucket_name: "{{env_name}}-artifacts" + object_name: "neo4j-community-3.3.9-unix.tar.gz" + src: /tmp/neo4j-community-3.3.9-unix.tar.gz + content_type: 'text/plain' + tags: + - oss + + - name: Downloading GeoIP2 + get_url: + url: "https://download.maxmind.com/app/geoip_download?edition_id={{ item.type }}&license_key={{ geoip_licence_key }}&suffix={{ item.prefix }}" + dest: "/tmp/geoip2_db.{{ item.prefix }}" + mode: '0440' + with_items: + - {type: 'GeoLite2-City-CSV', prefix: 'zip' } + - {type: 'GeoLite2-City', prefix: 'tar.gz'} + tags: + - oss + - name: Uploading GeoIP2 to artifacts + oci_object_storage_object: + namespace_name: "{{namespace_name}}" + bucket_name: "{{env_name}}-artifacts" + object_name: "geoip2_db.{{ item }}" + src: /tmp/geoip2_db.{{ item }} + content_type: 'text/plain' + with_items: + - zip + - tar.gz + tags: + - oss + + - name: Uploading labels + shell: + "oci os object bulk-upload --bucket-name {{env_name}}-label --prefix 'labels/' --src-dir ../utils/portal/labels/ --namespace {{ namespace_name }} -–overwrite" + tags: + - oss + + - name: Unarchive plugins + unarchive: + src: "https://sunbirdpublic.blob.core.windows.net/installation/content-plugins.zip" + dest: /tmp/ + creates: /tmp/content-plugins + remote_src: yes + tags: + - oss + + - name: Upload Content plugins + shell: + "oci os object bulk-upload --bucket-name {{env_name}}-contents --prefix 'content-plugins/' --src-dir /tmp/content-plugins/ --namespace {{ namespace_name }} " + tags: + - oss + + + - name: Create container_repository + oci_artifacts_container_repository: + compartment_id: "{{ compartment_id }}" + display_name: "{{repo_name}}/{{item}}" + is_public: false + readme: + content: "container registry for Sunbird deployment - {{item}} " + format: TEXT_MARKDOWN + with_items: "{{ repo_list }}" + tags: + - ocir + + - name: Launch private vm + oci_compute_instance: + availability_domain: "{{ availability_domain }}" + compartment_id: "{{ compartment_id }}" + display_name: "{{env_name}}-{{ item }}" + image_id: "{{ instance_image_id }}" + shape: "{{ instance_shape }}" + source_details: + boot_volume_size_in_gbs: "{{private_vm_hdd_in_gb}}" + image_id: "{{ instance_image_id }}" + source_type: "image" + shape_config: + ocpus: "{{ ocpu }}" + memory_in_gbs: "{{ memory_in_gbs }}" + create_vnic_details: + assign_public_ip: False + hostname_label: "{{env_name}}-{{ item }}" + subnet_id: "{{ private_subnet_id }}" + metadata: + ssh_authorized_keys: "{{ authorized_key }}" + user_data: "{{ lookup('file', '/tmp/oci_vms_bootstrap.sh' ) | b64encode }}" + key_by: [compartment_id, availability_domain, display_name] + with_items: "{{ private_vm_names }}" + register: private_vms + tags: + - vm + + - name: Launch jenkins instance + oci_compute_instance: + availability_domain: "{{ availability_domain }}" + compartment_id: "{{ compartment_id }}" + display_name: "{{env_name}}-{{ item }}" + image_id: "{{ instance_image_id }}" + shape: "{{ instance_shape }}" + source_details: + boot_volume_size_in_gbs: "{{public_vm_hdd_in_gb}}" + image_id: "{{ instance_image_id }}" + source_type: "image" + shape_config: + ocpus: "{{ ocpu }}" + memory_in_gbs: "{{ memory_in_gbs }}" + create_vnic_details: + assign_public_ip: True + hostname_label: "{{env_name}}-{{ item }}" + subnet_id: "{{ public_subnet_id }}" + metadata: + ssh_authorized_keys: "{{ authorized_key }}" + key_by: [compartment_id, availability_domain, display_name] + with_items: "{{ public_vm_names }}" + register: jenkins_vm + tags: + - vm + + - name: Create user + oci_identity_user: + compartment_id: "{{tenancy_id}}" + name: "{{svc_user_name}}" + description: sunbird ed service user + state: present + register: result + tags: + - identity + + - set_fact: + user_id: "{{ result.user.id }}" + cacheable: yes + tags: + - identity + + - name: Create group + oci_identity_group: + compartment_id: "{{tenancy_id}}" + name: "{{svc_user_group}}" + description: User group for Sunbird Service Users + state: present + register: result + tags: + - identity + - set_fact: + group_id: "{{ result.group.id }}" + cacheable: yes + tags: + - identity + + - name: Create user_group_membership + oci_identity_user_group_membership: + user_id: "{{user_id}}" + group_id: "{{group_id}}" + compartment_id: "{{tenancy_id}}" + state: present + tags: + - identity + + - name: Create dynamic group + oci_identity_dynamic_group: + compartment_id: "{{tenancy_id}}" + name: "{{dynamic_group_name}}" + description: Dynamic Group for Sunbird BB + matching_rule: "instance.compartment.id = '{{compartment_id}}'" + tags: + - identity + + - name: Create policy + oci_identity_policy: + compartment_id: "{{tenancy_id}}" + name: sunbirded-policy + description: Policy for Sunbird BB + statements: + - "Allow dynamic-group {{dynamic_group_name}} to manage all-resources in compartment id {{compartment_id}}" + - "Allow group {{svc_user_group}} to manage repos in compartment id {{compartment_id}}" + - "Allow group {{svc_user_group}} to manage buckets in compartment id {{compartment_id}}" + - "Allow group {{svc_user_group}} to manage buckets in compartment id {{compartment_id}}" + tags: + - identity + + - name: Create auth_token + oci_identity_auth_token: + description: For sunbird ocir + user_id: "{{user_id}}" + register: result + tags: + - identity + - set_fact: + auth_token: "{{ result.auth_token.token }}" + cacheable: yes + tags: + - identity + + - name: Create customer key + oci_identity_customer_secret_key: + display_name: For sunbird s3 operation + user_id: "{{user_id}}" + register: customer_key + tags: + - identity + - set_fact: + s3_access_key: "{{ customer_key.customer_secret_key.id }}" + s3_access_secret: "{{ customer_key.customer_secret_key.key }}" + cacheable: yes + tags: + - identity + + - name: Create cluster + oci_container_engine_cluster: + compartment_id: "{{compartment_id}}" + vcn_id: "{{vcn_id}}" + name: "{{env_name}}-sunbird-ed-oke" + kubernetes_version: "{{k8s_version}}" + endpoint_config: + subnet_id: "{{private_subnet_id}}" + is_public_ip_enabled: false + cluster_pod_network_options: + - cni_type: FLANNEL_OVERLAY + options: + service_lb_subnet_ids: [ "{{public_subnet_id}}" ] + type: BASIC_CLUSTER + register: oke + tags: + - k8s + - debug: + msg: "{{ oke }}" + tags: + - k8s + - set_fact: + cluster_id: "{{oke.cluster.id }}" + tags: + - k8s + + + - name: Create a node pool + oci_container_engine_node_pool: + cluster_id: "{{ cluster_id }}" + compartment_id: "{{ compartment_id }}" + name: "pool1" + ssh_public_key: "{{ authorized_key }}" + kubernetes_version: "{{ k8s_version }}" + node_source_details: "{{ node_pool_source_details }}" + node_shape: "{{ node_shape }}" + node_config_details: + size: 4 + placement_configs: + - availability_domain: "{{availability_domain}}" + subnet_id: "{{private_subnet_id}}" + node_pool_pod_network_option_details: + pod_subnet_ids: [ "{{private_subnet_id}}" ] + cni_type: FLANNEL_OVERLAY + node_shape_config: + ocpus: "{{ oke_ocpu }}" + memory_in_gbs: "{{ oke_memory_in_gbs }}" + register: np_result + tags: + - k8s + + - set_fact: + node_pool_id: "{{ np_result.node_pool.id }}" + tags: + - k8s + + + + - name: Get Private VMs Info + oci_compute_instance_facts: + compartment_id: "{{compartment_id}}" + display_name: "{{item}}" + lifecycle_state: RUNNING + with_items: "{{private_vm_names}}" + register: nics + tags: + - info + + - name: Get Jenkins VMs Info + oci_compute_instance_facts: + compartment_id: "{{compartment_id}}" + display_name: "jenkins" + lifecycle_state: RUNNING + register: jenkins_vm + tags: + - info + + - name: Showing resources info + debug: + msg: + - Jenkins public ip: "{{ jenkins_vm.instances[0].primary_public_ip }}" + - OCIR url: "{{ocir_region}}.ocir.io/{{namespace_name}}/{{repo_name}}" + - OCIR user: "{{namespace_name}}/{{svc_user_name}}" + - OCIR auth token: "{{ auth_token }}" + - S3 access key: "{{ s3_access_key }}" + - S3 access secret : "{{ s3_access_secret }}" + tags: + - info + + - name: Showing private vm ips + debug: + msg: + "VM {{ item.instances[0].display_name }} private ip: {{ item.instances[0].primary_private_ip }}" + with_items: "{{ nics.results }}" + loop_control: + label: "{{ item.instances[0].display_name }}" + tags: + - info + + - name: Writing resource info to oci-resources.txt + copy: + dest: /tmp/oci-resources.txt + content: |- + ================================================================ + jenkins public ip: {{ jenkins_vm.instances[0].primary_public_ip }} + OCIR url: "{{ocir_region}}.ocir.io/{{namespace_name}}/{{repo_name}}" + OCIR user: "{{namespace_name}}/{{svc_user_name}}" + OCIR auth token: "{{ auth_token }}" + S3 access key: "{{ s3_access_key }}" + S3 access secret : "{{ s3_access_secret }}" + tags: + - info + + - name: Writing machine ips to oci-resources.txt + lineinfile: + state: present + insertbefore: EOF + dest: /tmp/oci-resources.txt + line: "{{ item.instances[0].display_name }} private ip: {{ item.instances[0].primary_private_ip }}" + with_items: "{{ nics.results }}" + loop_control: + label: "{{ item.instances[0].display_name }}" + tags: + - info + + - debug: + msg: ansible variable values to update for the installation is stored in /tmp/oci-resources.txt" + tags: + - info \ No newline at end of file diff --git a/deploy/oci_vms_bootstrap.j2 b/deploy/oci_vms_bootstrap.j2 new file mode 100644 index 0000000000..f696b6bc41 --- /dev/null +++ b/deploy/oci_vms_bootstrap.j2 @@ -0,0 +1,3 @@ +#!/bin/bash +iptables -I INPUT -s {{vcn_cidr_block}} -j ACCEPT +iptables-save > /etc/iptables/rules.v4 \ No newline at end of file diff --git a/kubernetes/ansible/bootstrap.yaml b/kubernetes/ansible/bootstrap.yaml index 0fa7e0acfe..72f1beacff 100644 --- a/kubernetes/ansible/bootstrap.yaml +++ b/kubernetes/ansible/bootstrap.yaml @@ -39,7 +39,7 @@ - name: "Tagging {{ namespace }} to enable istio" shell: "kubectl label namespaces {{ namespace }} istio-injection=enabled --overwrite" - name: Creating docker secrets - shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ namespace }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password {{ vault_docker_registry_password }} --dry-run=client -o=yaml | kubectl apply -f -" + shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ namespace }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password '{{ vault_docker_registry_password }}' --dry-run=client -o=yaml | kubectl apply -f -" - name: Installing reloader for configmaps reload shell: helm upgrade --install --atomic reloader ../helm_charts/core/reloader --namespace "{{ namespace }}" -f /tmp/helm_vars.yaml roles: diff --git a/kubernetes/ansible/bootstrap_minimal.yaml b/kubernetes/ansible/bootstrap_minimal.yaml index 44170246ae..cdb145423e 100644 --- a/kubernetes/ansible/bootstrap_minimal.yaml +++ b/kubernetes/ansible/bootstrap_minimal.yaml @@ -30,7 +30,7 @@ - logging ignore_errors: yes - name: Creating docker secrets - shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password {{ vault_docker_registry_password }} --dry-run=client -o=yaml | kubectl apply -f -" + shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password '{{ vault_docker_registry_password }}' --dry-run=client -o=yaml | kubectl apply -f -" when: imagepullsecrets|length > 0 with_items: - "{{ bootstrap_namespace.split(',') }}" diff --git a/kubernetes/ansible/namespace.yml b/kubernetes/ansible/namespace.yml index bc62234c2c..bc38fab7d8 100644 --- a/kubernetes/ansible/namespace.yml +++ b/kubernetes/ansible/namespace.yml @@ -28,7 +28,7 @@ - "{{ bootstrap_namespace.split(',') }}" ignore_errors: yes - name: Creating docker secrets - shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password {{ vault_docker_registry_password }} --dry-run=client -o=yaml | kubectl apply -f -" + shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password '{{ vault_docker_registry_password }}' --dry-run=client -o=yaml | kubectl apply -f -" when: imagepullsecrets|length > 0 with_items: - "{{ bootstrap_namespace.split(',') }}" diff --git a/kubernetes/helm_charts/core/adminutils/templates/hpa.yaml b/kubernetes/helm_charts/core/adminutils/templates/hpa.yaml index 39a0140656..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/adminutils/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/adminutils/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,13 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} -{{- end }} - +{{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/analytics/templates/deployment.yaml b/kubernetes/helm_charts/core/analytics/templates/deployment.yaml index 0926360f76..57198cb77b 100644 --- a/kubernetes/helm_charts/core/analytics/templates/deployment.yaml +++ b/kubernetes/helm_charts/core/analytics/templates/deployment.yaml @@ -35,14 +35,14 @@ spec: value: {{ .Values.env.min_heap | quote }} - name: MAX_HEAP value: {{ .Values.env.max_heap | quote }} - - name: azure_storage_secret - value: {{ .Values.env.azure_private_account_secret | quote }} - - name: azure_storage_key - value: {{ .Values.env.azure_private_account_name | quote }} - - name: public_azure_storage_secret - value: {{ .Values.env.azure_public_account_secret | quote }} - - name: public_azure_storage_key - value: {{ .Values.env.azure_public_account_name | quote }} + - name: cloud_storage_secret + value: {{ .Values.env.cloud_private_account_secret | quote }} + - name: cloud_storage_key + value: {{ .Values.env.cloud_private_account_name | quote }} + - name: public_cloud_storage_secret + value: {{ .Values.env.cloud_public_account_secret | quote }} + - name: public_cloud_storage_key + value: {{ .Values.env.cloud_public_account_name | quote }} - name: _JAVA_OPTIONS value: -Dlog4j2.formatMsgNoLookups=true envFrom: diff --git a/kubernetes/helm_charts/core/analytics/templates/hpa.yaml b/kubernetes/helm_charts/core/analytics/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/analytics/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/analytics/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/analytics/values.j2 b/kubernetes/helm_charts/core/analytics/values.j2 index 277f840947..5aae4e1daa 100644 --- a/kubernetes/helm_charts/core/analytics/values.j2 +++ b/kubernetes/helm_charts/core/analytics/values.j2 @@ -8,10 +8,10 @@ env: javaoptions: {{analytics_java_mem_limit|default('-Xmx600m')}} min_heap: {{analytics_min_heap_limit|default('-Xms1g')}} max_heap: {{analytics_max_heap_limit|default('-Xmx2g')}} - azure_private_account_secret: {{ cloud_private_storage_secret }} - azure_private_account_name: {{ cloud_private_storage_accountname }} - azure_public_account_secret: {{ cloud_public_storage_secret }} - azure_public_account_name: {{ cloud_public_storage_accountname }} + cloud_private_account_secret: {{ cloud_private_storage_secret }} + cloud_private_account_name: {{ cloud_private_storage_accountname }} + cloud_public_account_secret: {{ cloud_public_storage_secret }} + cloud_public_account_name: {{ cloud_public_storage_accountname }} replicaCount: {{analytics_replicacount|default(1)}} repository: {{analytics_repository|default('sunbird-analytics-service')}} image_tag: {{ image_tag }} diff --git a/kubernetes/helm_charts/core/apimanager/templates/hpa.yaml b/kubernetes/helm_charts/core/apimanager/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/apimanager/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/apimanager/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/apimanagerecho/templates/hpa.yaml b/kubernetes/helm_charts/core/apimanagerecho/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/apimanagerecho/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/apimanagerecho/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/assessment/templates/hpa.yaml b/kubernetes/helm_charts/core/assessment/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/assessment/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/assessment/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/cert/templates/hpa.yaml b/kubernetes/helm_charts/core/cert/templates/hpa.yaml index 1ba4f89a8b..8447d79f56 100644 --- a/kubernetes/helm_charts/core/cert/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/cert/templates/hpa.yaml @@ -1,5 +1,6 @@ + {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +17,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/kubernetes/helm_charts/core/certregistry/templates/hpa.yaml b/kubernetes/helm_charts/core/certregistry/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/certregistry/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/certregistry/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/content/templates/hpa.yaml b/kubernetes/helm_charts/core/content/templates/hpa.yaml index 1ba4f89a8b..450417d31f 100644 --- a/kubernetes/helm_charts/core/content/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/content/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/kubernetes/helm_charts/core/dial/templates/hpa.yaml b/kubernetes/helm_charts/core/dial/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/dial/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/dial/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/discussionsmw/templates/hpa.yaml b/kubernetes/helm_charts/core/discussionsmw/templates/hpa.yaml index cb527919ea..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/discussionsmw/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/discussionsmw/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/enc/templates/hpa.yaml b/kubernetes/helm_charts/core/enc/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/enc/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/enc/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/gotenberg/templates/hpa.yaml b/kubernetes/helm_charts/core/gotenberg/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/gotenberg/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/gotenberg/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/gql/templates/hpa.yaml b/kubernetes/helm_charts/core/gql/templates/hpa.yaml index 0fef8ca0e7..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/gql/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/gql/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} - {{- end }} \ No newline at end of file +{{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/groups/templates/hpa.yaml b/kubernetes/helm_charts/core/groups/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/groups/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/groups/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/inbound/templates/hpa.yaml b/kubernetes/helm_charts/core/inbound/templates/hpa.yaml index 0fef8ca0e7..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/inbound/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/inbound/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} - {{- end }} \ No newline at end of file +{{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/knowledgemw/templates/hpa.yaml b/kubernetes/helm_charts/core/knowledgemw/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/knowledgemw/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/knowledgemw/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/learner/templates/hpa.yaml b/kubernetes/helm_charts/core/learner/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/learner/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/learner/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/lms/templates/hpa.yaml b/kubernetes/helm_charts/core/lms/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/lms/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/lms/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/ml-analytics-service/.helmignore b/kubernetes/helm_charts/core/ml-analytics-service/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/core/ml-analytics-service/Chart.yaml b/kubernetes/helm_charts/core/ml-analytics-service/Chart.yaml new file mode 100644 index 0000000000..eb2395299a --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: ml-analytics-service +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/_helpers.tpl b/kubernetes/helm_charts/core/ml-analytics-service/templates/_helpers.tpl new file mode 100644 index 0000000000..3f238f7ff7 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "ml-analytics-service.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ml-analytics-service.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ml-analytics-service.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "ml-analytics-service.labels" -}} +helm.sh/chart: {{ include "ml-analytics-service.chart" . }} +{{ include "ml-analytics-service.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "ml-analytics-service.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ml-analytics-service.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ml-analytics-service.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "ml-analytics-service.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/configmap.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/configmap.yaml new file mode 100644 index 0000000000..5f03f92832 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/configmap.yaml @@ -0,0 +1,11 @@ +--- +{{- $keys := .Files.Glob "keys/*" }} +{{ if $keys }} +apiVersion: v1 +kind: Secret +metadata: + name: ml-analytics-access-keys +type: Opaque +data: +{{ (.Files.Glob "keys/*").AsSecrets | indent 2 }} +{{ end }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml new file mode 100644 index 0000000000..d28628b7e5 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + volumes: + - name: {{ .Chart.Name }}-config + configMap: + name: {{ .Chart.Name }}-config + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + volumeMounts: + - name: {{ .Chart.Name }}-config + mountPath: /opt/sparkjobs/ml-analytics-service/config.ini + subPath: ml-analytics-service.conf diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/service.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/service.yaml new file mode 100644 index 0000000000..4b47ed5332 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/values.j2 b/kubernetes/helm_charts/core/ml-analytics-service/values.j2 new file mode 100644 index 0000000000..2a2a1c8638 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/values.j2 @@ -0,0 +1,21 @@ +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{ml_analytics_replicacount|default(1)}} +repository: {{ml_analytics_repository|default('ml-analytics-service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{ml_analytics_cpu_req|default('1000m')}} + memory: {{ml_analytics_mem_req|default('1000Mi')}} + limits: + cpu: {{ml_analytics_cpu_limit|default('2')}} + memory: {{ml_analytics_mem_limit|default('2048Mi')}} +network: + port: 7000 + targetport: 7000 +strategy: + type: RollingUpdate + maxsurge: {{ ml_core_maxsurge|default('25%') }} + maxunavailable: {{ ml_core_maxunavailable|default('25%') }} diff --git a/kubernetes/helm_charts/core/ml-core-service/templates/hpa.yaml b/kubernetes/helm_charts/core/ml-core-service/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/ml-core-service/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/ml-core-service/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/ml-projects-service/templates/hpa.yaml b/kubernetes/helm_charts/core/ml-projects-service/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/ml-projects-service/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/ml-projects-service/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/ml-reports-service/templates/hpa.yaml b/kubernetes/helm_charts/core/ml-reports-service/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/ml-reports-service/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/ml-reports-service/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/ml-survey-service/templates/hpa.yaml b/kubernetes/helm_charts/core/ml-survey-service/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/ml-survey-service/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/ml-survey-service/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml b/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml index b59cebe9a0..4cb69fd90f 100644 --- a/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml +++ b/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml @@ -40,7 +40,7 @@ data: listen 80; listen [::]:80; {{- if and .Values.nginx_private_ingress_ip (ne .Values.csp "oci") }} - server_name: {{ .Values.nginx_private_ingress_ip }}; + server_name {{ .Values.nginx_private_ingress_ip }}; {{- end }} resolver {{ .Values.kube_dns_ip }}; diff --git a/kubernetes/helm_charts/core/nginx-private-ingress/templates/hpa.yaml b/kubernetes/helm_charts/core/nginx-private-ingress/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/nginx-private-ingress/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/nginx-private-ingress/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/templates/configMap.yaml b/kubernetes/helm_charts/core/nginx-public-ingress/templates/configMap.yaml index 0f7f0dcc16..3a04ccb80c 100644 --- a/kubernetes/helm_charts/core/nginx-public-ingress/templates/configMap.yaml +++ b/kubernetes/helm_charts/core/nginx-public-ingress/templates/configMap.yaml @@ -6,6 +6,10 @@ metadata: data: proxy-default.conf: | {{ .Values.proxyconfig | indent 4 }} +{{- if eq .Values.csp "oci" }} + cors-proxy-default.conf: | +{{ .Values.corsproxyconfig | indent 4 }} +{{- end }} compression.conf: | {{ .Values.compressionConfig | indent 4 }} diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 b/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 index e325f5d339..11e33d70c4 100644 --- a/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 +++ b/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 @@ -1,6 +1,7 @@ #jinja2:lstrip_blocks: True namespace: {{ namespace }} +csp: {{cloud_service_provider}} merge_domain_status: {{ merge_domain_status | lower }} service: annotations: {{nginx_public_ingress_service_annotations | d('') | to_json}} @@ -64,6 +65,74 @@ resources: repository: {{proxy_repository|default('proxy')}} image_tag: {{ image_tag }} +corsproxyconfig: |- + {% if proto=='https' %} + server { + if ($host = files.{{domain_name}}) { + return 301 https://$host$request_uri; + } + listen 80 ; + listen [::]:80 ; + server_name files.{{domain_name}}; + return 404; + } + {% endif %} + server { + {% if proto=='http' %} + listen 80; + listen [::]:80; + {% else %} + listen [::]:443 ssl ipv6only=on; + listen 443 ssl; + ssl_certificate /etc/secrets/site.crt; + ssl_certificate_key /etc/secrets/site.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA HIGH !RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS"; + {% endif %} + server_name files.{{domain_name}}; + client_max_body_size 0; + root /var/www/html; + resolver {{ kube_dns_ip }} valid=30s; + + location / { + # handle cors and allow all + if ($request_method = OPTIONS ) { + add_header Access-Control-Allow-Origin *; + add_header Access-Control-Allow-Methods "GET, OPTIONS, PATCH, POST, PUT, HEAD"; + add_header Access-Control-Allow-Headers "Access-Control-Allow-Origin, Authorization, Content-Type, user-id, Accept, Accept-Encoding, Accept-Language, Access-Control-Request-Headers, Access-Control-Request-Method, Cache-Control, DNT, User-Agent, X-Amz-Algorithm, X-Amz-Credential, X-Amz-Date, Amz-Expires, X-Amz-SignedHeaders, X-Amz-Signature, x-ms-blob-type"; + add_header Access-Control-Allow-Credentials "true"; + add_header Content-Length 0; + add_header Content-Type text/plain; + return 204; + } + + proxy_set_header Host "{{ cloud_storage_url | replace('https://', '') }}"; + # remove any CORS header from backend OSS S3 + proxy_hide_header Access-Control-Allow-Origin; + proxy_hide_header Access-Control-Allow-Methods; + proxy_hide_header Access-Control-Allow-Headers; + proxy_hide_header Access-Control-Allow-Credentials; + + # inject our own CORS header to allow what we wanted + add_header Access-Control-Allow-Credentials "true" always; + add_header Access-Control-Expose-Headers 'Content-Length,Content-Range,Connection,opc-client-info,opc-request-id' always; + add_header Access-Control-Allow-Origin * always; + add_header Access-Control-Allow-Methods "GET,OPTIONS,PATCH,POST,PUT,HEAD" always; + add_header Access-Control-Allow-Headers "Access-Control-Allow-Origin, Authorization, Content-Type, user-id, Accept,Accept-Encoding,Accept-Language, Access-Control-Request-Headers, Access-Control-Request-Method,Cache-Control,DNT,Host,Origin,Pragma,Referer,User-Agent, X-Amz-Algorithm, X-Amz-Credential, X-Amz-Date, Amz-Expires, X-Amz-SignedHeaders, X-Amz-Signature, x-ms-blob-type" always; + # + add_header Referer ""; + proxy_pass {{cloud_storage_url}}; + + # if get request, trim the query string + if ($request_method = GET ) { + proxy_pass {{cloud_storage_url}}$uri; + } + + + } + } + + proxyconfig: |- {% if proto=='https' %} server { diff --git a/kubernetes/helm_charts/core/nodebb/templates/hpa.yaml b/kubernetes/helm_charts/core/nodebb/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/nodebb/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/nodebb/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/notification/templates/hpa.yaml b/kubernetes/helm_charts/core/notification/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/notification/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/notification/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/odk/templates/hpa.yaml b/kubernetes/helm_charts/core/odk/templates/hpa.yaml index 0fef8ca0e7..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/odk/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/odk/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} - {{- end }} \ No newline at end of file +{{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/orchestrator/templates/hpa.yaml b/kubernetes/helm_charts/core/orchestrator/templates/hpa.yaml index 0fef8ca0e7..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/orchestrator/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/orchestrator/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} - {{- end }} \ No newline at end of file +{{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/outbound/templates/hpa.yaml b/kubernetes/helm_charts/core/outbound/templates/hpa.yaml index 0fef8ca0e7..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/outbound/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/outbound/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} - {{- end }} \ No newline at end of file +{{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/player/templates/hpa.yaml b/kubernetes/helm_charts/core/player/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/player/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/player/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/print/templates/hpa.yaml b/kubernetes/helm_charts/core/print/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/print/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/print/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/report/templates/hpa.yaml b/kubernetes/helm_charts/core/report/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/report/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/report/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/search/templates/hpa.yaml b/kubernetes/helm_charts/core/search/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/search/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/search/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/taxonomy/templates/hpa.yaml b/kubernetes/helm_charts/core/taxonomy/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/taxonomy/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/taxonomy/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/telemetry/templates/hpa.yaml b/kubernetes/helm_charts/core/telemetry/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/telemetry/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/telemetry/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/transformer/templates/hpa.yaml b/kubernetes/helm_charts/core/transformer/templates/hpa.yaml index 0fef8ca0e7..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/transformer/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/transformer/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} - {{- end }} \ No newline at end of file +{{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/uci/templates/hpa.yaml b/kubernetes/helm_charts/core/uci/templates/hpa.yaml index 0fef8ca0e7..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/uci/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/uci/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} - {{- end }} \ No newline at end of file +{{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/userorg/templates/hpa.yaml b/kubernetes/helm_charts/core/userorg/templates/hpa.yaml index 1ba4f89a8b..fd75ba3280 100644 --- a/kubernetes/helm_charts/core/userorg/templates/hpa.yaml +++ b/kubernetes/helm_charts/core/userorg/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/autoscale.yaml b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/autoscale.yaml index 2455ac3450..e000b80855 100755 --- a/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/autoscale.yaml +++ b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/autoscale.yaml @@ -1,7 +1,7 @@ {{- range $key, $spec := .Values }} {{- if ne $key "enabled" }} {{- if and $spec.enabled $spec.autoscaleEnabled $spec.autoscaleMin $spec.autoscaleMax }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ $key }} @@ -28,4 +28,4 @@ spec: --- {{- end }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/autoscale.yaml b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/autoscale.yaml index 377b47d033..77f150dbea 100755 --- a/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/autoscale.yaml +++ b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/autoscale.yaml @@ -1,7 +1,7 @@ {{- range $key, $spec := .Values }} {{- if or (eq $key "policy") (eq $key "telemetry") }} {{- if and $spec.enabled $spec.autoscaleEnabled $spec.autoscaleMin $spec.autoscaleMax }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: istio-{{ $key }} @@ -26,4 +26,4 @@ spec: --- {{- end }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/autoscale.yaml b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/autoscale.yaml index 1a9945136a..223eb03995 100755 --- a/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/autoscale.yaml +++ b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/autoscale.yaml @@ -1,5 +1,5 @@ {{- if and .Values.autoscaleEnabled .Values.autoscaleMin .Values.autoscaleMax }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: istio-pilot @@ -22,4 +22,4 @@ spec: name: cpu targetAverageUtilization: {{ .Values.cpu.targetAverageUtilization }} --- -{{- end }} +{{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/sunbird-RC/certificateapi/templates/hpa.yaml b/kubernetes/helm_charts/sunbird-RC/certificateapi/templates/hpa.yaml index cb527919ea..fd75ba3280 100644 --- a/kubernetes/helm_charts/sunbird-RC/certificateapi/templates/hpa.yaml +++ b/kubernetes/helm_charts/sunbird-RC/certificateapi/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/sunbird-RC/certificatesign/templates/hpa.yaml b/kubernetes/helm_charts/sunbird-RC/certificatesign/templates/hpa.yaml index cb527919ea..fd75ba3280 100644 --- a/kubernetes/helm_charts/sunbird-RC/certificatesign/templates/hpa.yaml +++ b/kubernetes/helm_charts/sunbird-RC/certificatesign/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/sunbird-RC/registry/templates/hpa.yaml b/kubernetes/helm_charts/sunbird-RC/registry/templates/hpa.yaml index cb527919ea..fd75ba3280 100644 --- a/kubernetes/helm_charts/sunbird-RC/registry/templates/hpa.yaml +++ b/kubernetes/helm_charts/sunbird-RC/registry/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ .Chart.Name }} @@ -16,12 +16,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/pipelines/upload/schema/inquiry/Jenkinsfile b/pipelines/upload/schema/inquiry/Jenkinsfile new file mode 100644 index 0000000000..f37147a563 --- /dev/null +++ b/pipelines/upload/schema/inquiry/Jenkinsfile @@ -0,0 +1,55 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + } + + ansiColor('xterm') { + values = [:] + currentWs = sh(returnStdout: true, script: 'pwd').trim() + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + stage('deploy artifact'){ + sh """ + rm -rf schema_repo + git clone ${params.schema_repo} -b ${params.schema_repo_branch_or_tag} schema_repo + """ + ansiblePlaybook = "${currentWs}/ansible/kp_upload-schema.yml" + ansibleExtraArgs = "--extra-vars \" source_name=${currentWs}/schema_repo/schemas \" --vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = 'SUCCESS' + currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}, schema_repo_branch_or_tag: ${params.schema_repo_branch_or_tag}" + } + } + summary() + } + catch (err) { + currentBuild.result = 'FAILURE' + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} diff --git a/private_repo/ansible/inventory/dev/Core/common.yml b/private_repo/ansible/inventory/dev/Core/common.yml index 3386020d08..78e201f3bd 100644 --- a/private_repo/ansible/inventory/dev/Core/common.yml +++ b/private_repo/ansible/inventory/dev/Core/common.yml @@ -10,21 +10,25 @@ private_ingressgateway_ip: "" # your private kubernetes load balancer ip # Cloud Service Provider Variables # If cloud_service_provider is AWS then update with access key as value # Example: cloud_public_storage_accountname: "AKSHKSJHFJDHJDSHFKSD" +# If cloud_service_provider is OCI then update with access key as value +# Example: cloud_public_storage_accountname: "7855461d26e1d8f211a04ec59cfd3037c8d1c6dd" # If cloud_service_provider is gloud(GCP) then update with service account name # Example: cloud_public_storage_accountname: "cloudstorage-gcp-test.iam.gserviceaccount.com" # If cloud_service_provider is AZURE then update with stoage account name # Example: cloud_public_storage_accountname: "azurestotageaccount" cloud_public_storage_accountname: "" -# If cloud_service_provider is AWS then update with region +# If cloud_service_provider is AWS/OCI then update with region # Example: cloud_public_storage_region: us-east-1 cloud_public_storage_region: "" # If cloud_service_provider is gcp then update this variable with project id # Example: cloud_public_storage_project: "sunbird-gcp-test" cloud_public_storage_project: "" - +# If cloud_service_provider is OCI then update this variable with namespace value +# Example: cloud_public_storage_namespace: "apsyjjclx" +cloud_public_storage_namespace: "" # Create object storage for each below mentioned variables and update accordingly -# If cloud_service_provider is AWS update with bucket name +# If cloud_service_provider is AWS/OCI update with bucket name # If cloud_service_provider is gcloud(GCP) update with bucket name # If cloud_service_provider is AZURE update with container name # Example: cloud_storage_certqr_bucketname: "certqr-storage" @@ -96,6 +100,8 @@ cloud_storage_management_bucketname: "" # cloud_storage_url: https://storage.googleapis.com # AWS # cloud_storage_url: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" +# OCI +# cloud_storage_url: "https://{{ cloud_public_storage_namespace }}.compat.objectstorage.{{ cloud_public_storage_region }}.oraclecloud.com" # Azure cloud_storage_url: "https://{{ cloud_public_storage_accountname }}.blob.core.windows.net" @@ -106,6 +112,18 @@ cloud_storage_url: "https://{{ cloud_public_storage_accountname }}.blob.core.win # for google: # nginx_private_ingress_annotations: # cloud.google.com/load-balancer-type: "Internal" +# for OCI private +#nginx_private_ingress_annotations: +# service.beta.kubernetes.io/oci-load-balancer-internal: "true" +# service.beta.kubernetes.io/oci-load-balancer-subnet1: "ocid1.subnet.oc1.ap-hyderabad-1.xxxxxxxxxxxx" #subnet id +# for OCI public +# nginx public - oci specific +#nginx_public_ingress_service_annotations: +# oci.oraclecloud.com/load-balancer-type: "lb" +# service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" +# service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "100Mbps" +# service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "200Mbps" +# service.beta.kubernetes.io/oci-load-balancer-subnet1: "ocid1.subnet.oc1.ap-hyderabad-1.xxxxxxxxxxx" #subnet id # for azure: # nginx_private_ingress_annotations: # service.beta.kubernetes.io/azure-load-balancer-internal: "true" @@ -113,10 +131,26 @@ cloud_storage_url: "https://{{ cloud_public_storage_accountname }}.blob.core.win nginx_private_ingress_annotations: service.beta.kubernetes.io/azure-load-balancer-internal: "true" +# Media specific vars - OCI ( enable when cloud_service_provider is oci) +#oci_media_region: "{{ cloud_public_storage_region }}" +#oci_media_compartment: ocid1.compartment.oc1..xxxxxxxxxxxx # compartment id +#oci_media_namespace: "{{ cloud_public_storage_namespace }}" +#oci_media_source_bucket: "{{ cloud_storage_content_bucketname }}" +#oci_media_target_bucket: "{{ cloud_storage_content_bucketname }}" +#oci_media_prefix_input: "" +#oci_media_dist_channel_id: "" +#oci_media_work_flow_id: "" +#oci_media_stream_config_id: "" +#oci_media_gateway_domain: "" +#csp_migrator_router_parallelism: "" # ------------------------------------------------------------------------------------------------------------ # graylog_trusted_proxies: "1.2.3.4/21" # This should be the kubernetes nodes subnet CIDR range +### Android Mobile deeplink url - Upload the assetlinks.json file to the public object store (bucket) +## reference: https://project-sunbird.atlassian.net/wiki/spaces/SBDES/pages/3276865567/Deep+link+support+in+Android+12+devices +mobile_deeplink_url: "{{ cloud_storage_url }}/{{ cloud_storage_public_bucketname }}/Mobile-deeplink/assetlinks.json" + ## Optional learningservice_ip: "10.0.1.5" # Load balancer IP or server ip @@ -481,10 +515,30 @@ sunbird_kong_device_register_anonymous: 'true' # enables anonymous sessions sunbird_session_store_type: redis # uses redis for session data instead of cassandra portal_redis_connection_string: "redis://:@{{ sunbird_redis_host }}:6379/3" # Uses KP redis and DB number 3 to store session data +#### ML Core Service ##### +#Secure token to access Admin-related APIs.The value of the internal access token can be generated by command - "openssl rand -hex 10" +ml_core_internal_access_token: 8c3a94f0934e01a4940a # THIS IS A REFERENCE VALUE ONLY. + #### ML Analytics Service ##### ml_analytics_cname_url: # Reference value - https://obj.dev.sunbirded.org/samiksha/ +## ML internal kong #### +internal_kong_postgres_host: +internal_kong_ssl: True +internal_kong_postgres_user: +internal_kong_postgres_password: +internal_kong_postgres_database: + ### Variables added as part of 6.0.0 release - https://github.com/Sunbird-Ed/ml-analytics-service/tree/release-6.0.0/release-notes/6.0.0.md ml_analytics_client_id : project-sunbird-dev-client # Keycloak client group ml_analytics_username : reportAdmin1 # Username of a user which has roles of "PROGRAM_MANAGER", "PROGRAM_DESIGNER", "REPORT_ADMIN", "REPORT_VIEWER" ml_analytics_createdBy : fb85a044-d9eb-479b-a55a-faf1bfaea14d # Unique system generated user UUID which is the same user as above +ml_container: samiksha +ml_kafka_topic_create: "" # set to true if ml related topics needs to be created +# provide the s3 compatible endpoint +# for AWS +# cloud_private_storage_account_endpoint: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" +# for OCI +#cloud_private_storage_account_endpoint: "https://.compat.objectstorage.{{cloud_public_storage_region}}.oraclecloud.com" +cloud_private_storage_account_endpoint: "{{ cloud_public_storage_endpoint }}" # Leave Blank for Azure +bb_name: "" #Leave Blank diff --git a/private_repo/ansible/inventory/dev/Core/secrets.yml b/private_repo/ansible/inventory/dev/Core/secrets.yml index 2e16859454..e511b72aac 100644 --- a/private_repo/ansible/inventory/dev/Core/secrets.yml +++ b/private_repo/ansible/inventory/dev/Core/secrets.yml @@ -5,7 +5,7 @@ core_vault_postgres_password: "change.postgres.password" # Update same core_vault_dp_pgdb_password: "change.postgres.password" #update same password in dp secrets dp_vault_pgdb_password variable core_vault_sunbird_azure_storage_key: "change.azure.storage.account.key" core_vault_docker_registry_url: "change.docker.url" # for docker hub "https://index.docker.io/v1" -core_vault_docker_registry_user: "change.docker.username" +core_vault_docker_registry_user: "change.docker.username" # For OCI oci_namespcae/oci_username core_vault_docker_registry_password: "change.docker.password" # Cloud Service Provider Secret Variables @@ -31,7 +31,7 @@ core_vault_docker_registry_password: "change.docker.password" # Example: cloud_management_storage_secret: "BfoOQwA6UFI7URwkFjkjdjsdhjhjhjsdhjfM//lsTbXDmHOFR5GqEwAwzyrKXOUDywojCpOC/g==" cloud_management_storage_secret: "" -# If cloud_service_provider is aws then update secret access key +# If cloud_service_provider is aws/oci then update secret access key # Example: cloud_public_storage_secret: "xkxjfjsdsfjdfjdhgjfhgjdhfgjdgjdfgdjgjdgdjhgjhd" # If cloud_service_provider is gcp then update with private-key which is in service account json file @@ -238,3 +238,7 @@ lp_vault_youtube_api_key: # youtube api token if you want ml_analytics_authorization_access_token : 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJkOTY3NzRjYzXXXXXXXXXXXXXQ4Y2RiOWQ2Mzg0OSJ9.n4hXxKxl_698yeZPSWhXeGvMOb4esfgPadIZe8jZ0Z4' # Bearer auth token which has access to APIs as defined in the release note ml_analytics_password : Test@123 # Password of a user which has roles of "PROGRAM_MANAGER", "PROGRAM_DESIGNER", "REPORT_ADMIN", "REPORT_VIEWER" matching with supplied username to variable ml_analytics_username ml_analytics_client_secret : fd241dce-4ZZ9-47e1-97cf-1c7de7a44216 # Keycloak client secret for creating tokens, mapped to client id supplied to variable ml_analytics_client_id + +#### ML Core Service ##### +ml_core_internal_access_token: 8c3a94f0934e01a4940a # THIS IS A REFERENCE VALUE ONLY.Secure token to access Admin-related APIs.The value of the internal access token can be generated by command - "openssl rand -hex 10" +ml_api_access_token: 8c3a94f0934e01a4940a # THIS IS A REFERENCE VALUE ONLY.Secure token to access Admin-related APIs.The value of the internal access token can be generated by command - "openssl rand -hex 10" diff --git a/private_repo/ansible/inventory/dev/DataPipeline/common.yml b/private_repo/ansible/inventory/dev/DataPipeline/common.yml index c5790ec42c..fa568ffdf2 100644 --- a/private_repo/ansible/inventory/dev/DataPipeline/common.yml +++ b/private_repo/ansible/inventory/dev/DataPipeline/common.yml @@ -6,25 +6,30 @@ domain_name: "" # your domain name like example.com dockerhub: "change.docker.url" # docker hub username or url incase of private registry private_ingressgateway_ip: "" # your private kubernetes load balancer ip search_lb_ip: # search service Load balancer IP +learningservice_ip: "10.0.1.5" # Load balancer IP or server ip # Cloud Service Provider Variables # If cloud_service_provider is AWS then update with access key as value # Example: cloud_public_storage_accountname: "AKSHKSJHFJDHJDSHFKSD" +# If cloud_service_provider is OCI then update with access key as value +# Example: cloud_public_storage_accountname: "7855461d26e1d8f211a04ec59cfd3037c8d1c6dd" # If cloud_service_provider is gloud(GCP) then update with service account name # Example: cloud_public_storage_accountname: "cloudstorage-gcp-test.iam.gserviceaccount.com" # If cloud_service_provider is AZURE then update with stoage account name # Example: cloud_public_storage_accountname: "azurestotageaccount" + cloud_public_storage_accountname: "" -# If cloud_service_provider is AWS then update with region +# If cloud_service_provider is AWS/OCI then update with region # Example: cloud_public_storage_region: us-east-1 cloud_public_storage_region: "" # If cloud_service_provider is gcp then update this variable with project id # Example: cloud_public_storage_project: "sunbird-gcp-test" cloud_public_storage_project: "" - - +# If cloud_service_provider is OCI then update this variable with namespace value +# Example: cloud_public_storage_namespace: "apsyjjclx" +cloud_public_storage_namespace: "" # Create object storage for each below mentioned variables and update accordingly -# If cloud_service_provider is AWS update with bucket name +# If cloud_service_provider is AWS/OCI update with bucket name # If cloud_service_provider is gcloud(GCP) update with bucket name # If cloud_service_provider is AZURE update with container name # Example: cloud_storage_certqr_bucketname: "certqr-storage" @@ -96,6 +101,8 @@ cloud_storage_management_bucketname: "" # cloud_storage_url: https://storage.googleapis.com # AWS # cloud_storage_url: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" +# OCI +# cloud_storage_url: "https://{{ cloud_public_storage_namespace }}.compat.objectstorage.{{ cloud_public_storage_region }}.oraclecloud.com" # Azure cloud_storage_url: "https://{{ cloud_public_storage_accountname }}.blob.core.windows.net" @@ -130,6 +137,11 @@ postgres: db_port: 5432 db_admin_user: postgres db_admin_password: "{{dp_vault_pgdb_admin_password}}" + dp_ssl: false + dp_ssl_mode: require + dp_sslfactory: org.postgresql.ssl.NonValidatingFactory + dp_db_name: analytics + spark_db_name: analytics druid_postgres_user: druid # Do not change this imagepullsecrets: "{{env}}registrysecret" # kubernetes imagePullSecrets @@ -178,6 +190,11 @@ cloudstorage_base_path: "{{ cloud_storage_url }}" valid_cloudstorage_base_urls: '["{{ cloud_storage_url }}"]' cloudstorage_relative_path_prefix: "CONTENT_STORAGE_BASE_PATH" +cloud_storage_telemetry_type: "{{ cloud_service_provider }}" +druid_storage_type: +spark_bds_url: +druid_cloud_storage_pathstyle_access: true +spark_version: # overide this var if you need a different version of spark. Ex: spark_version: 3.1.3 # The below sets the kafka topics retention time to 1 day, if you use the defaults from the public repo, it will be 7 days # If you want to retain the topics for 7 days, remove the below sections completely # Ensure you have atleast 1 TB of disk to retain data for 7 days @@ -295,4 +312,4 @@ processing_kafka_overriden_topics: replication_factor: 1 # graylog -send_logs_to_graylog: true # filebeat agents will send logs to graylog instead of ES \ No newline at end of file +send_logs_to_graylog: true # filebeat agents will send logs to graylog instead of ES diff --git a/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml b/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml index ce4e93b37e..5c13f9d9b9 100644 --- a/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml +++ b/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml @@ -5,16 +5,18 @@ cloud_service_provider: "" # Your cloud service provider name. Supported v dockerhub: "change.docker.url" # docker hub username or url incase of private registry private_ingressgateway_ip: "" # your private kubernetes load balancer ip domain_name: "" # your domain name like example.com +kp_print_service_base_url: "http://{{private_ingressgateway_ip}}/print" +learningservice_ip: "10.0.1.5" # Load balancer IP or server ip # Cloud Service Provider Variables -# If cloud_service_provider is AWS then update with access key as value +# If cloud_service_provider is AWS/OCI then update with access key as value # Example: cloud_public_storage_accountname: "AKSHKSJHFJDHJDSHFKSD" # If cloud_service_provider is gloud(GCP) then update with service account name # Example: cloud_public_storage_accountname: "cloudstorage-gcp-test.iam.gserviceaccount.com" # If cloud_service_provider is AZURE then update with stoage account name # Example: cloud_public_storage_accountname: "azurestotageaccount" cloud_public_storage_accountname: "" -# If cloud_service_provider is AWS then update with region +# If cloud_service_provider is AWS/OCI then update with region # Example: cloud_public_storage_region: us-east-1 cloud_public_storage_region: "" # If cloud_service_provider is gcp then update this variable with project id @@ -26,7 +28,7 @@ cloud_public_storage_namespace: "" # Create object storage for each below mentioned variables and update accordingly -# If cloud_service_provider is AWS update with bucket name +# If cloud_service_provider is AWS/OCI update with bucket name # If cloud_service_provider is gcloud(GCP) update with bucket name # If cloud_service_provider is AZURE update with container name # Example: cloud_storage_certqr_bucketname: "certqr-storage" @@ -98,6 +100,8 @@ cloud_storage_management_bucketname: "" # cloud_storage_url: https://storage.googleapis.com # AWS # cloud_storage_url: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" +# OCI +# cloud_storage_url: "https://{{ cloud_public_storage_namespace }}.compat.objectstorage.{{ cloud_public_storage_region }}.oraclecloud.com" # Azure cloud_storage_url: "https://{{ cloud_public_storage_accountname }}.blob.core.windows.net" @@ -129,8 +133,11 @@ kubeconfig_path: /var/lib/jenkins/secrets/k8s.yaml # kubeconfig file path # cloud_public_storage_endpoint: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" # for OCI #cloud_public_storage_endpoint: "https://.compat.objectstorage.{{cloud_public_storage_region}}.oraclecloud.com" - -cloud_public_storage_endpoint: "" +#oci_flink_s3_storage_endpoint: "{{ cloud_public_storage_endpoint }}" +#cloudstorage_sdk_endpoint: "{{ cloud_public_storage_endpoint }}" +#s3_region: "{{ cloud_public_storage_region }}" +#cloud_storage_proxy_host: "{{cloud_storage_url}}" +#flink_container_name: "{{ cloud_storage_flink_bucketname }}" # Update below vars if seperate object storage is required cloud_private_storage_accountname: "{{ cloud_public_storage_accountname }}" @@ -163,13 +170,26 @@ cloud_artifact_storage_namespace: "{{ cloud_public_storage_namespace }}" # cloud_storage_neo4jbackup_bucketname: "" # cloud_storage_redisbackup_bucketname: "" +# Media specific vars - OCI ( enable when cloud_service_provider is oci) +#oci_media_region: "{{ cloud_public_storage_region }}" +#oci_media_compartment: ocid1.compartment.oc1..xxxxxxxxxxxx # compartment id +#oci_media_namespace: "{{ cloud_public_storage_namespace }}" +#oci_media_source_bucket: "{{ cloud_storage_content_bucketname }}" +#oci_media_target_bucket: "{{ cloud_storage_content_bucketname }}" +#oci_media_prefix_input: "" +#oci_media_dist_channel_id: "" +#oci_media_work_flow_id: "" +#oci_media_stream_config_id: "" +#oci_media_gateway_domain: "" +#csp_migrator_router_parallelism: "" + # Building block vars cloud_storage_base_url: "{{ cloud_storage_url }}" cloudstorage_base_path: "{{ cloud_storage_url }}" valid_cloudstorage_base_urls: '["{{ cloud_storage_url }}"]' cloudstorage_relative_path_prefix: "CONTENT_STORAGE_BASE_PATH" -cloud_storage_pathstyle_access: false - +cloud_storage_pathstyle_access: true +cloud_storage_cname_url: "{{ cloud_storage_url }}" # overide if you have seperate url for cname ### Lern BB - Adding Lern specific vars here. In future if we want to move it to seperate folder this can be used as the starting point # Mandatorty variables - DO NOT LEAVE ANYTHING BLANK # @@ -213,4 +233,10 @@ postgres: db_admin_password: "{{dp_vault_pgdb_admin_password}}" # graylog -send_logs_to_graylog: true # filebeat agents will send logs to graylog instead of ES \ No newline at end of file +send_logs_to_graylog: true # filebeat agents will send logs to graylog instead of ES + +druid_storage_type: "" # Your cloud service provider name. Supported values are aws, azure, gcloud + +# Optional variables - Can be left blank +cloud_storage_report_verfication_bucketname: "" +dp_storage_endpoint_config: ""