diff --git a/.gitignore b/.gitignore
index 9fcc736131..cb3eca673d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,3 +29,4 @@ target/
*.iml
**/.settings
dependency-reduced-pom.xml
+.vscode
diff --git a/ansible/artifacts-download.yml b/ansible/artifacts-download.yml
index 9db0efb42f..6b8629cc4e 100644
--- a/ansible/artifacts-download.yml
+++ b/ansible/artifacts-download.yml
@@ -21,6 +21,8 @@
name: gcp-cloud-storage
tasks_from: download.yml
vars:
+ gcp_storage_service_account_name: "{{ cloud_artifact_storage_accountname }}"
+ gcp_storage_key_file: "{{ cloud_artifact_storage_secret }}"
gcp_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
gcp_path: "{{ artifact }}"
local_file_or_folder_path: "{{ artifact_path }}"
@@ -38,3 +40,17 @@
aws_access_key_id: "{{ cloud_artifact_storage_accountname }}"
aws_secret_access_key: "{{ cloud_artifact_storage_secret }}"
when: cloud_service_provider == "aws"
+
+
+ - name: download artifact from oci oss
+ include_role:
+ name: oci-cloud-storage
+ apply:
+ environment:
+ OCI_CLI_AUTH: "instance_principal"
+ tasks_from: download.yml
+ vars:
+ local_file_or_folder_path: "{{ artifact_path }}"
+ oss_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+ oss_object_name: "{{ artifact }}"
+ when: cloud_service_provider == "oci"
diff --git a/ansible/artifacts-upload.yml b/ansible/artifacts-upload.yml
index 3bdbe73017..c11ec36c3c 100644
--- a/ansible/artifacts-upload.yml
+++ b/ansible/artifacts-upload.yml
@@ -22,6 +22,8 @@
name: gcp-cloud-storage
tasks_from: upload.yml
vars:
+ gcp_storage_service_account_name: "{{ cloud_artifact_storage_accountname }}"
+ gcp_storage_key_file: "{{ cloud_artifact_storage_secret }}"
gcp_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
gcp_path: "{{ artifact }}"
local_file_or_folder_path: "{{ artifact_path }}"
@@ -39,3 +41,16 @@
aws_access_key_id: "{{ cloud_artifact_storage_accountname }}"
aws_secret_access_key: "{{ cloud_artifact_storage_secret }}"
when: cloud_service_provider == "aws"
+
+ - name: upload artifact to oci oss
+ include_role:
+ name: oci-cloud-storage
+ apply:
+ environment:
+ OCI_CLI_AUTH: "instance_principal"
+ tasks_from: upload.yml
+ vars:
+ local_file_or_folder_path: "{{ artifact_path }}"
+ oss_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+ oss_path: "{{ artifact }}"
+ when: cloud_service_provider == "oci"
\ No newline at end of file
diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml
index 13114d01d9..d11ccd88f0 100644
--- a/ansible/inventory/env/group_vars/all.yml
+++ b/ansible/inventory/env/group_vars/all.yml
@@ -6,12 +6,12 @@ log4j_appender_kafka_topic: "{{env}}.telemetry.backend"
# Shall we change the value to telemetry-data-store in dev also?
# What's the implication
# Is it azure blob or s3 bucket??
-channel_data_exhaust_bucket: dev-data-store
-secrets_path: '{{inventory_dir}}/secrets.yml'
-artifacts_container: "{{dp_vault_artifacts_container}}"
+cloud_storage_telemetry_bucketname: dev-data-store
+secrets_path: "{{inventory_dir}}/secrets.yml"
+cloud_storage_artifacts_bucketname: "{{dp_vault_artifacts_container}}"
-report_azure_account_name: "{{sunbird_private_storage_account_name}}"
-report_azure_storage_secret: "{{sunbird_private_storage_account_key}}"
+report_azure_account_name: "{{cloud_private_storage_accountname}}"
+report_azure_storage_secret: "{{cloud_private_storage_secret}}"
redis_host: "{{ groups['redis'][0] }}"
metadata_redis_host: "{{ groups['redis'][0] }}"
@@ -34,7 +34,7 @@ telemetry_schema_directory: /etc/{{env}}/telemetry
telemetry_schema_path: /etc/{{env}}/telemetry/schemas
schema_repo_url: https://github.com/project-sunbird/sunbird-data-pipeline.git
# Create learningall group with LP ips
-cassandra_host: "{{ groups['cassandra'][0] }}"
+cassandra_host: "{{ groups['cassandra'][0] }}"
core_cassandra_host: "{{ groups['core-cassandra'][0] }}"
lp_cassandra_host: "{{ groups['lp-cassandra'][0] }}"
report_cassandra_host: "{{ groups['report-cassandra'][0] }}"
@@ -46,36 +46,45 @@ sbin_path: "{{ analytics_user_home }}/sbin"
# Secor vars
secor:
- properties: ['secor.azure', 'secor.common', 'secor', 'secor.partition', 'log4j']
- artifact_dir: /mount/secor
- artifact_ver: "0.29"
- azure:
- account_name: "{{sunbird_private_storage_account_name}}"
- account_key: "{{sunbird_private_storage_account_key}}"
- container_name: "{{channel_data_exhaust_bucket}}"
- paths: ['/mount/secor', '/mount/secor/reports', '/mount/secor/logs', '/home/analytics/sbin', '/mount/data/analytics']
- channel: "{{secor_alerts_slack_channel}}"
+ properties:
+ ["secor.azure", "secor.common", "secor", "secor.partition", "log4j"]
+ artifact_dir: /mount/secor
+ artifact_ver: "0.29"
+ azure:
+ account_name: "{{cloud_private_storage_accountname}}"
+ account_key: "{{cloud_private_storage_secret}}"
+ container_name: "{{cloud_storage_telemetry_bucketname}}"
+
+ paths:
+ [
+ "/mount/secor",
+ "/mount/secor/reports",
+ "/mount/secor/logs",
+ "/home/analytics/sbin",
+ "/mount/data/analytics",
+ ]
+ channel: "{{secor_alerts_slack_channel}}"
# postgres
# list of databases to be created
# Can move this dictionary to postgres role; but incase we want to generalize roles!!
postgresql_databases:
- - name: analytics
- owner: analytics
+ - name: analytics
+ owner: analytics
postgresql_users:
- - name: analytics
- password: "{{dp_vault_pgdb_password}}"
+ - name: analytics
+ password: "{{dp_vault_pgdb_password}}"
postgres:
- db_url: "{{ groups['postgres'][0] }}"
- db_username: analytics
- db_name: analytics
- db_password: "{{dp_vault_pgdb_password}}"
- db_table_name: "{{env}}_consumer_channel_mapping"
- db_port: 5432
- db_admin_user: analytics
- db_admin_password: "{{dp_vault_pgdb_admin_password}}"
+ db_url: "{{ groups['postgres'][0] }}"
+ db_username: analytics
+ db_name: analytics
+ db_password: "{{dp_vault_pgdb_password}}"
+ db_table_name: "{{env}}_consumer_channel_mapping"
+ db_port: 5432
+ db_admin_user: analytics
+ db_admin_password: "{{dp_vault_pgdb_admin_password}}"
postgres_address_space: 0.0.0.0/0 # Postgres trust address space
@@ -84,9 +93,9 @@ lp_composite_search_host: "{{ groups['composite-search-cluster'][0] }}"
lp_search: "http://{{private_ingressgateway_ip}}/search"
lp_url: http://{{ groups['learning'][0] }}:8080/learning-service
service:
- search:
- url: http://{{private_ingressgateway_ip}}/search
- path: /v3/search
+ search:
+ url: http://{{private_ingressgateway_ip}}/search
+ path: /v3/search
cassandra_hierarchy_store_prefix: "{{env}}_"
data_exhaust_token: "{{dp_vault_data_exhaust_token}}"
@@ -106,46 +115,46 @@ CONTAINER_NAME_SAMZA: samza-logs
script_path: /usr/local/hadoop
job_names:
- DeDuplication_1:
- job_file_name: 'de-duplication'
- DeNormalization_1:
- job_file_name: 'de-normalization'
- DruidEventsValidator_1:
- job_file_name: 'druid-events-validator'
- EventsRouter_1:
- job_file_name: 'events-router'
- TelemetryExtractor_1:
- job_file_name: 'telemetry-extractor'
- TelemetryLocationUpdater_1:
- job_file_name: 'telemetry-location-updater'
- TelemetryRouter_1:
- job_file_name: 'telemetry-router'
- TelemetryRedacter_1:
- job_file_name: 'telemetry-redacter'
- TelemetryValidator_1:
- job_file_name: 'telemetry-validator'
- DeviceProfileUpdater_1:
- job_file_name: 'device-profile-updater'
- AssessmentAggregator_1:
- job_file_name: 'assessment-aggregator'
- DerivedDeDuplication_1:
- job_file_name: 'derived-de-duplication'
- UserCacheUpdater_1:
- job_file_name: 'user-cache-updater'
- ContentCacheUpdater_1:
- job_file_name: 'content-cache-updater'
- ShareEventsFlattener_1:
- job_file_name: 'share-events-flattener'
+ DeDuplication_1:
+ job_file_name: "de-duplication"
+ DeNormalization_1:
+ job_file_name: "de-normalization"
+ DruidEventsValidator_1:
+ job_file_name: "druid-events-validator"
+ EventsRouter_1:
+ job_file_name: "events-router"
+ TelemetryExtractor_1:
+ job_file_name: "telemetry-extractor"
+ TelemetryLocationUpdater_1:
+ job_file_name: "telemetry-location-updater"
+ TelemetryRouter_1:
+ job_file_name: "telemetry-router"
+ TelemetryRedacter_1:
+ job_file_name: "telemetry-redacter"
+ TelemetryValidator_1:
+ job_file_name: "telemetry-validator"
+ DeviceProfileUpdater_1:
+ job_file_name: "device-profile-updater"
+ AssessmentAggregator_1:
+ job_file_name: "assessment-aggregator"
+ DerivedDeDuplication_1:
+ job_file_name: "derived-de-duplication"
+ UserCacheUpdater_1:
+ job_file_name: "user-cache-updater"
+ ContentCacheUpdater_1:
+ job_file_name: "content-cache-updater"
+ ShareEventsFlattener_1:
+ job_file_name: "share-events-flattener"
druid_ingestion_specs:
- telemetry-events:
- ingestion_file_name: 'telemetry_index_kafka'
- summary-events:
- ingestion_file_name: 'summary_index_kafka'
- telemtry-feedback-events:
- ingestion_file_name: 'telemetry_feedback_index_kafka'
- pipeline-metrics:
- ingestion_file_name: 'pipeline_metrics_index_kafka'
+ telemetry-events:
+ ingestion_file_name: "telemetry_index_kafka"
+ summary-events:
+ ingestion_file_name: "summary_index_kafka"
+ telemtry-feedback-events:
+ ingestion_file_name: "telemetry_feedback_index_kafka"
+ pipeline-metrics:
+ ingestion_file_name: "pipeline_metrics_index_kafka"
#Druid Proxy APi service
sunbird_druid_broker_host: "http://{{ groups['raw-broker'][0] }}"
@@ -154,9 +163,9 @@ sunbird_learner_service_url: "http://{{private_ingressgateway_ip}}/learner"
location_search_url: "{{ domain_name }}/api/data/"
location_search_token: "Bearer {{ sunbird_api_auth_token }}"
-druid_report_url_endpoint : "{{ proto}}://{{domain_name}}/api/data/v1/report/jobs"
-druid_report_url : "{{ proto}}://{{domain_name}}/api/data/v1/"
-druid_report_token : "Bearer {{ sunbird_api_auth_token }}"
+druid_report_url_endpoint: "{{ proto}}://{{domain_name}}/api/data/v1/report/jobs"
+druid_report_url: "{{ proto}}://{{domain_name}}/api/data/v1/"
+druid_report_token: "Bearer {{ sunbird_api_auth_token }}"
#redis multiprocess config
content_port: 6379
diff --git a/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/templates/job-cluster-jobmanager.yaml b/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/templates/job-cluster-jobmanager.yaml
index 1e7c5e2778..a8773232bd 100644
--- a/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/templates/job-cluster-jobmanager.yaml
+++ b/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/templates/job-cluster-jobmanager.yaml
@@ -21,6 +21,7 @@ spec:
workingDir: /opt/flink
command: ["/opt/flink/bin/standalone-job.sh"]
{{- $job-config-key := .Release.Name }}
+
args: ["start-foreground",
"--job-classname={{ index .Values $job-config-key.job_classname }}",
"-Djobmanager.rpc.address={{ .Release.Name }}-jobmanager",
@@ -29,7 +30,20 @@ spec:
"-Dblob.server.port=6124",
"-Dqueryable-state.server.ports=6125",
"-Djobmanager.heap.size={{ index .Values $job-config-key.job_manager_heap_size }}",
- "-Dfs.azure.account.key.{{ .Values.azure_storage_account }}.blob.core.windows.net: {{ .Values.azure_storage_secret }}",
+{{- if eq .Values.checkpoint_store_type "azure" }}
+ "-Dfs.azure.account.key.{{ .Values.cloud_storage_key }}.blob.core.windows.net: {{ .Values.cloud_storage_secret }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "aws" }}
+ "-Ds3.access-key={{ .Values.cloud_storage_key }}",
+ "-Ds3.secret-key={{ .Values.cloud_storage_secret }}",
+ "-Ds3.endpoint={{ .Values.cloud_private_endpoint }}",
+ "-Ds3.path.style.access={{ .Values.s3_path_style_access }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "gcloud" }}
+ "-Dfs.gs.auth.client.id={{ .Values.cloud_storage_key }}",
+ "-Dfs.gs.auth.client.secret={{ .Values.cloud_storage_secret }}",
+ "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}"
+{{- end }}
"-Dconfig.file=/opt/flink/conf/{{ .Release.Name }}.conf"]
ports:
- containerPort: 6123
diff --git a/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/templates/job-cluster-taskmanager.yaml b/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/templates/job-cluster-taskmanager.yaml
index cad7c3f47c..0c858aaca2 100644
--- a/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/templates/job-cluster-taskmanager.yaml
+++ b/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/templates/job-cluster-taskmanager.yaml
@@ -29,7 +29,20 @@ spec:
"-Djobmanager.rpc.address={{ .Release.Name }}-jobmanager",
"-Dtaskmanager.rpc.port=6122",
"-Dtaskmanager.heap.size={{ index .Values $job-config-key.task_manager_heap_size }}",
- "-Dfs.azure.account.key.{{ .Values.azure_storage_account }}.blob.core.windows.net: {{ .Values.azure_storage_secret }}",
+{{- if eq .Values.checkpoint_store_type "azure" }}
+ "-Dfs.azure.account.key.{{ .Values.cloud_storage_key }}.blob.core.windows.net: {{ .Values.cloud_storage_secret }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "aws" }}
+ "-Ds3.access-key={{ .Values.cloud_storage_key }}",
+ "-Ds3.secret-key={{ .Values.cloud_storage_secret }}",
+ "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}",
+ "-Ds3.path.style.access={{ .Values.cloud_storage_path_style_access }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "gcloud" }}
+ "-Dfs.gs.auth.client.id={{ .Values.cloud_storage_key }}",
+ "-Dfs.gs.auth.client.secret={{ .Values.cloud_storage_secret }}",
+ "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}",
+{{- end }}
"-Dconfig.file=/opt/flink/conf/{{ .Release.Name }}.conf"]
ports:
- containerPort: 6122
diff --git a/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/values.j2 b/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/values.j2
index fd34c8c647..d7c53a44c7 100644
--- a/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/values.j2
+++ b/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/values.j2
@@ -6,8 +6,12 @@ dockerhub: {{ dockerhub }}
repository: {{ datapipeline_repository|default('data-pipeline') }}
image_tag: {{ image_tag }}
-azure_storage_account={{ sunbird_private_storage_account_name }}
-azure_storage_secret={{ sunbird_private_storage_account_key }}
+checkpoint_store_type: {{ cloud_service_provider }}
+cloud_storage_key: {{ cloud_private_storage_accountname }}
+cloud_storage_secret: {{ cloud_private_storage_secret }}
+cloud_storage_endpoint: {{ cloud_private_storage_endpoint }}
+cloud_storage_path_style_access: {{ cloud_storage_pathstyle_access }}
+cloud_storage_project_id: {{ cloud_private_storage_project }}
telemetry-extractor:
job_name=telemetry-extractor
diff --git a/ansible/lpa_data-products_deploy.yml b/ansible/lpa_data-products_deploy.yml
index 7e84ba53f7..903afcc7b3 100644
--- a/ansible/lpa_data-products_deploy.yml
+++ b/ansible/lpa_data-products_deploy.yml
@@ -5,7 +5,7 @@
become: yes
become_user: "{{ analytics_user }}"
environment:
- AZURE_STORAGE_ACCOUNT: "{{sunbird_private_storage_account_name}}"
- AZURE_STORAGE_KEY: "{{sunbird_private_storage_account_key}}"
+ AZURE_STORAGE_ACCOUNT: "{{cloud_private_storage_accountname}}"
+ AZURE_STORAGE_KEY: "{{cloud_private_storage_secret}}"
roles:
- - data-products-deploy
+ - data-products-deploy
diff --git a/ansible/oci-bds-spark.provision.yml b/ansible/oci-bds-spark.provision.yml
new file mode 100644
index 0000000000..4eb80aa296
--- /dev/null
+++ b/ansible/oci-bds-spark.provision.yml
@@ -0,0 +1,18 @@
+- hosts: local
+ become: yes
+ vars_files:
+ - "{{inventory_dir}}/secrets.yml"
+ roles:
+ - oci-bds-spark-cluster
+ tags:
+ - copy-script
+
+- hosts: bds-livy-node
+ become: yes
+ gather_facts: no
+ vars_files:
+ - "{{inventory_dir}}/secrets.yml"
+ roles:
+ - provision-oci-spark-cluster
+ tags:
+ - spark-provision
diff --git a/ansible/roles/analytics-bootstrap-always/meta/main.yml b/ansible/roles/analytics-bootstrap-always/meta/main.yml
index af15826aef..3566881eff 100644
--- a/ansible/roles/analytics-bootstrap-always/meta/main.yml
+++ b/ansible/roles/analytics-bootstrap-always/meta/main.yml
@@ -1,6 +1,7 @@
---
dependencies:
- { role: jdk11 , become: yes }
- - { role: azure-cli , become: yes }
+ - { role: azure-cli , become: yes, when: cloud_service_provider == "azure" }
+ - { role: oci-cli , become: yes, when: cloud_service_provider == "oci" }
diff --git a/ansible/roles/analytics-bootstrap-always/tasks/main.yml b/ansible/roles/analytics-bootstrap-always/tasks/main.yml
index 065a3b3afb..da30d5e5da 100644
--- a/ansible/roles/analytics-bootstrap-always/tasks/main.yml
+++ b/ansible/roles/analytics-bootstrap-always/tasks/main.yml
@@ -17,6 +17,10 @@
createhome: yes
group: "{{ analytics_group }}"
+- name: change permission
+ become: yes
+ command: chown -R "{{ analytics_user }}":"{{ analytics_user }}" /home/"{{ analytics_user }}"
+
- name: Installing packages
become: yes
diff --git a/ansible/roles/analytics-bootstrap-spark/tasks/main.yml b/ansible/roles/analytics-bootstrap-spark/tasks/main.yml
index 13ba75f78a..a26365f691 100644
--- a/ansible/roles/analytics-bootstrap-spark/tasks/main.yml
+++ b/ansible/roles/analytics-bootstrap-spark/tasks/main.yml
@@ -4,34 +4,35 @@
become: yes
become_user: "{{ analytics_user }}"
lineinfile:
- path: '{{ analytics_user_home }}/.bashrc'
- line: 'export {{item.var}}={{item.value}}'
+ path: "{{ analytics_user_home }}/.bashrc"
+ line: "export {{item.var}}={{item.value}}"
regexp: "export {{ item.var }}.*"
with_items:
- - {var: 'azure_storage_key', value: '{{ sunbird_private_storage_account_name }}'}
- - {var: 'azure_storage_secret', value: '{{ sunbird_private_storage_account_key }}'}
+ - {var: 'azure_storage_key', value: '{{ cloud_private_storage_accountname }}'}
+ - {var: 'azure_storage_secret', value: '{{ cloud_private_storage_secret }}'}
- name: Adding ENV Vars to spark servers environment.
become: yes
lineinfile:
- path: '/etc/environment'
- line: '{{item.var}}={{item.value}}'
+ path: "/etc/environment"
+ line: "{{item.var}}={{item.value}}"
regexp: "{{ item.var }}.*"
with_items:
- - {var: 'azure_storage_key', value: '{{ sunbird_private_storage_account_name }}'}
- - {var: 'azure_storage_secret', value: '{{ sunbird_private_storage_account_key }}'}
- - {var: 'AZURE_STORAGE_ACCOUNT', value: '{{ sunbird_private_storage_account_name }}'}
- - {var: 'AZURE_STORAGE_ACCESS_KEY', value: '{{ sunbird_private_storage_account_key }}'}
- - {var: 'PUBLIC_AZURE_STORAGE_ACCOUNT', value: '{{ sunbird_public_storage_account_name }}'}
- - {var: 'PUBLIC_AZURE_STORAGE_ACCESS_KEY', value: '{{ sunbird_public_storage_account_key }}'}
- - {var: 'PRIVATE_REPORT_CONTAINER', value: '{{ sunbird_private_azure_report_container_name }}'}
- - {var: 'PUBLIC_REPORT_CONTAINER', value: '{{ sunbird_public_azure_report_container_name }}'}
+ - {var: 'azure_storage_key', value: '{{ cloud_private_storage_accountname }}'}
+ - {var: 'azure_storage_secret', value: '{{ cloud_private_storage_secret }}'}
+ - {var: 'AZURE_STORAGE_ACCOUNT', value: '{{ cloud_private_storage_accountname }}'}
+ - {var: 'AZURE_STORAGE_ACCESS_KEY', value: '{{ cloud_private_storage_secret }}'}
+ - {var: 'PUBLIC_AZURE_STORAGE_ACCOUNT', value: '{{ cloud_public_storage_accountname }}'}
+ - {var: 'PUBLIC_AZURE_STORAGE_ACCESS_KEY', value: '{{ cloud_public_storage_secret }}'}
+ - {var: 'PRIVATE_REPORT_CONTAINER', value: '{{ cloud_storage_privatereports_bucketname }}'}
+ - {var: 'PUBLIC_REPORT_CONTAINER', value: '{{ cloud_storage_publicreports_bucketname }}'}
- {var: 'REPORT_BACKUP_CONTAINER', value: 'portal-reports-backup'}
- {var: 'GOOGLE_CREDENTIALS_PATH', value: '/home/analytics/credentials'}
- {var: 'STORAGE_PROVIDER', value: 'AZURE'}
- {var: 'ENV', value: '{{env}}'}
- {var: 'KAFKA_BROKER_HOST', value: "{{groups['processing-cluster-kafka'][0]}}:9092"}
+
- name: Install required python packages
become: yes
action: apt pkg={{ item }} state=present update_cache=yes
diff --git a/ansible/roles/analytics-druid/defaults/main.yml b/ansible/roles/analytics-druid/defaults/main.yml
index 8a55bc3f97..9c160b25b2 100644
--- a/ansible/roles/analytics-druid/defaults/main.yml
+++ b/ansible/roles/analytics-druid/defaults/main.yml
@@ -20,30 +20,29 @@ druid_zookeeper_host: "{{ groups[cluster+'-zookeeper']|join(':2181,')}}:2181"
druid_postgres_user: druid
#druid_postgres_host: "{{ groups['druid-postgres'][0] }}"
-
druid_default_tmp_dir: "/var/tmp"
druid_gc_logdir: "/var/log/druid/crash_logs"
druid_crash_logdir: "/var/log/druid/crash_logs"
druid_log_dir: "/var/log/druid/"
-#Writing request query logs to file
+#Writing request query logs to file
druid_request_logging_type: "file"
#Druid Extensions
-druid_storage_type: "azure"
+druid_storage_type: "{{ cloud_storage_telemetry_type }}"
-druid_extensions_list : '"druid-azure-extensions", "graphite-emitter", "postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-datasketches"'
+druid_extensions_list: '"druid-azure-extensions", "graphite-emitter", "postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-datasketches"'
druid_community_extensions:
- - graphite-emitter
+ - graphite-emitter
# End of druid_extensions
-druid_indexing_logs_type: azure
+druid_indexing_logs_type: "{{ cloud_storage_telemetry_type }}"
druid_indexing_log_dir: /var/druid/indexing-logs
-druid_indexing_storage_type : metadata
-druid_indexing_task_basedir : "/var/task"
+druid_indexing_storage_type: metadata
+druid_indexing_task_basedir: "/var/task"
druid_common_monitors: '"com.metamx.metrics.JvmMonitor","org.apache.druid.java.util.metrics.JvmMonitor"'
druid_common_emitters: '"logging","graphite"'
@@ -56,14 +55,14 @@ druid_whitelist_filepath: "{{ druid_path }}whitelist"
#Coordinator Configurations
druid_coordinator_port: 8081
-druid_coordinator_service : druid/coordinator
+druid_coordinator_service: druid/coordinator
druid_coordinator_tmp_dir: "{{ druid_default_tmp_dir }}"
druid_coordinator_gc_logfile: "{{ druid_crash_logdir }}/gc.log"
#Overlord Configurations
druid_overlord_port: 8090
-druid_overlord_service : druid/overlord
+druid_overlord_service: druid/overlord
druid_overlord_heap_size: 256m
druid_overlord_tmp_dir: "{{ druid_default_tmp_dir }}"
@@ -82,14 +81,13 @@ druid_broker_gc_logdir: "{{ druid_crash_logdir }}/gc.log"
druid_historical_port: 8083
druid_historical_service: druid/historical
-
druid_historical_tmp_dir: "{{ druid_default_tmp_dir }}"
druid_historical_gc_logfile: "{{ druid_crash_logdir }}/historical.gc.log"
druid_historical_heap_dump_file: "{{ druid_crash_logdir }}/historical.hprof"
-druid_broker_heap_dump_file : "{{ druid_crash_logdir }}/broker.hprof"
-druid_coordinator_heap_dump_file : "{{ druid_crash_logdir }}/coordinator.hprof"
-druid_overlord_heap_dump_file : "{{ druid_crash_logdir }}/overlord.hprof"
-druid_mm_heap_dump_file : "{{ druid_crash_logdir }}/middlemanager.hprof"
+druid_broker_heap_dump_file: "{{ druid_crash_logdir }}/broker.hprof"
+druid_coordinator_heap_dump_file: "{{ druid_crash_logdir }}/coordinator.hprof"
+druid_overlord_heap_dump_file: "{{ druid_crash_logdir }}/overlord.hprof"
+druid_mm_heap_dump_file: "{{ druid_crash_logdir }}/middlemanager.hprof"
druid_historical_monitoring_monitors: '"org.apache.druid.server.metrics.HistoricalMetricsMonitor","com.metamx.metrics.JvmMonitor"'
@@ -116,152 +114,165 @@ druid_router_tmp_dir: "{{ druid_default_tmp_dir }}"
druid_router_gc_logfile: "{{ druid_crash_logdir }}/gc.router.log"
druid_router_heap_dump_file: "{{ druid_crash_logdir }}/router.hprof"
-
default_druid_configs:
- raw:
- #Druid Postgres Details
- druid_postgres_db: "druid"
- druid_postgres_host: "{{ postgres.db_url }}"
- druid_postgres_port: "{{ postgres.db_port }}"
- druid_postgres_user: "{{ druid_postgres_user | default('druid@' + postgres.db_url) }}"
- #Druid Azure Details
- druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}"
- azure_account_name: "{{ sunbird_druid_storage_account_name }}"
- azure_storage_secret: "{{ sunbird_druid_storage_account_key }}"
- azure_container: "{{ druid_azure_container_name }}"
- #Logging the indexing logs to azure
- druid_log_azure_container: "{{ druid_azure_container_name }}"
- druid_log_azure_folder: "druidlogs"
- #Druid S3 Details
- druid_storage_type: "{{ druid_storage_type }}"
- s3_access_key: "{{ s3_storage_key }}"
- s3_secret_key: "{{ s3_storage_secret }}"
- s3_bucket: "{{ s3_storage_container }}"
- s3_endpoint: "{{ s3_storage_endpoint }}"
- s3_segment_dir: "druid/raw/segments"
- s3_path_like_access: "{{ s3_path_style_access }}"
- s3_v4_sign_region: "{{ s3_default_bucket_location }}"
- #Logging the indexing logs to s3
- s3_logging_bucket: "{{ s3_storage_container }}"
- s3_indexer_logs_dir: "druid/raw/stage/indexing_logs"
- #Druid coordinator node configuration
- druid_coordinator_heap_size: 128m
- druid_coordinator_period: PT30S
- druid_coordinator_startDelay: PT30S
- druid_coordinator_balance_strategy: diskNormalized
- #Druid overlord node configuration
- druid_overlord_heap_size: 256m
- #Druid broker node configuration
- druid_broker_min_heap_size: 128m
- druid_broker_max_heap_size: 128m
- druid_broker_max_direct_size: 800m
- druid_broker_http_numConnections: 5
- druid_broker_server_http_numThread: 25
- druid_broker_processing_bufferBytes: 134217728
- druid_broker_processing_threads: 2
- #Druid historical node configuration
- druid_historical_min_heap_size: 1048m
- druid_historical_max_heap_size: 1048m
- druid_historical_max_direct_size: 800m
- druid_historical_http_numConnections: 5
- druid_historical_server_http_numThread: 25
- druid_historical_processing_bufferBytes: 134217728
- druid_historical_processing_threads: 2
- druid_historical_enable_cache: false
- druid_historical_segmentcache_size: 10000000000
- druid_historical_server_maxsize: 10000000000
- druid_historical_processing_num_merge_buffers: 2
- druid_query_ondiskstorage_enabled: true
- druid_historical_maxMergingDictionarySize: 100000000
- druid_historical_segmentcache_numloadingthreads: 4
- druid_historical_segmentcache_path: "/var/segmentstore"
- druid.query.groupBy.maxOnDiskStorage: 10737418240
- #Druid middlemanager configuration
- druid_middlemanager_heap_size: 128m
- druid_middlemanager_worker_cap: 4
- druid_mm_java_opts_array: "-server -Xmx900m -XX:+UseG1GC -XX:HeapDumpPath={{ druid_mm_heap_dump_file }} -XX:MaxGCPauseMillis=100 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps"
- druid_middlemanager_peon_server_http_numThread: 25
- druid_middlemanager_peon_processing_bufferBytes: 25000000
- druid_middlemanager_peon_processing_threads: 2
- druid_middlemanager_peon_server_maxsize: 0
- druid_indexing_queue_startDelay: PT30S
- druid_router_heap_size: 1g
- druid_router_http_numConnections: 50
- druid_router_http_readTimeout: PT5M
- druid_router_http_numMaxThreads: 100
- druid_server_http_numThreads: 100
- druid_router_managementProxy_enabled: true
- druid_historical_maxOnDiskStorage: 10737418240
- rollup:
- #Druid Postgres Details
- druid_postgres_db: "druid"
- druid_postgres_host: "{{ postgres.db_url }}"
- druid_postgres_port: "{{ postgres.db_port }}"
- druid_postgres_user: "{{ druid_postgres_user | default('druid@' + postgres.db_url) }}"
- #Druid Azure Details
- druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}"
- azure_account_name: "{{ sunbird_druid_storage_account_name }}"
- azure_storage_secret: "{{ sunbird_druid_storage_account_key }}"
- azure_container: "{{ druid_azure_container_name }}"
- #Logging the indexing logs to azure
- druid_log_azure_container: "{{ druid_azure_container_name }}"
- druid_log_azure_folder: "druidlogs"
- #Druid S3 Details
- druid_storage_type: "{{ druid_storage_type }}"
- s3_access_key: "{{ s3_storage_key }}"
- s3_secret_key: "{{ s3_storage_secret }}"
- s3_bucket: "{{ s3_storage_container }}"
- s3_endpoint: "{{ s3_storage_endpoint }}"
- s3_segment_dir: "druid/rollup/segments"
- s3_path_like_access: "{{ s3_path_style_access }}"
- s3_v4_sign_region: "{{ s3_default_bucket_location }}"
- #Logging the indexing logs to s3
- s3_logging_bucket: "{{ s3_storage_container }}"
- s3_indexer_logs_dir: "druid/rollup/stage/indexing_logs"
- #Druid coordinator node configuration
- druid_coordinator_heap_size: 128m
- druid_coordinator_period: PT30S
- druid_coordinator_startDelay: PT30S
- druid_coordinator_balance_strategy: cost
- #Druid overlord node configuration
- druid_overlord_heap_size: 256m
- #Druid broker node configuration
- druid_broker_min_heap_size: 128m
- druid_broker_max_heap_size: 128m
- druid_broker_max_direct_size: 700m
- druid_broker_http_numConnections: 5
- druid_broker_server_http_numThread: 25
- druid_broker_processing_bufferBytes: 134217728
- druid_broker_processing_threads: 2
- #Druid historical node configuration
- druid_historical_min_heap_size: 1048m
- druid_historical_max_heap_size: 1048m
- druid_historical_max_direct_size: 800m
- druid_historical_http_numConnections: 5
- druid_historical_server_http_numThread: 25
- druid_historical_processing_bufferBytes: 134217728
- druid_historical_processing_threads: 2
- druid_historical_enable_cache: false
- druid_historical_segmentcache_size: 2000000000
- druid_historical_server_maxsize: 10000000000
- druid_historical_processing_num_merge_buffers: 2
- druid_query_ondiskstorage_enabled: false
- druid_historical_segmentcache_numloadingthreads: 4
- druid_historical_segmentcache_path: "/var/segmentstore"
- #Druid middlemanager configuration
- druid_middlemanager_heap_size: 128m
- druid_middlemanager_worker_cap: 4
- druid_mm_java_opts_array: "-server -Xmx900m -XX:+UseG1GC -XX:HeapDumpPath={{ druid_mm_heap_dump_file }} -XX:MaxGCPauseMillis=100 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps"
- druid_middlemanager_peon_server_http_numThread: 25
- druid_middlemanager_peon_processing_bufferBytes: 25000000
- druid_middlemanager_peon_processing_threads: 2
- druid_middlemanager_peon_server_maxsize: 0
- druid_indexing_queue_startDelay: PT30S
- druid_router_heap_size: 1g
- druid_router_http_numConnections: 50
- druid_router_http_readTimeout: PT5M
- druid_router_http_numMaxThreads: 100
- druid_server_http_numThreads: 100
- druid_router_managementProxy_enabled: true
+ raw:
+ #Druid Postgres Details
+ druid_postgres_db: "druid"
+ druid_postgres_host: "{{ postgres.db_url }}"
+ druid_postgres_port: "{{ postgres.db_port }}"
+ druid_postgres_user: "{{ druid_postgres_user | default('druid@' + postgres.db_url) }}"
+ #Druid Azure Details
+ druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}"
+ azure_account_name: "{{ cloud_private_storage_accountname }}"
+ azure_storage_secret: "{{ cloud_private_storage_secret }}"
+ azure_container: "{{ cloud_storage_telemetry_bucketname }}"
+ #Logging the indexing logs to azure
+ druid_log_azure_container: "{{ cloud_storage_telemetry_bucketname }}"
+ druid_log_azure_folder: "druidlogs"
+ #Druid S3 Details
+ druid_storage_type: "{{ cloud_storage_telemetry_type }}"
+ s3_access_key: "{{ cloud_private_storage_accountname }}"
+ s3_secret_key: "{{ cloud_private_storage_secret }}"
+ s3_bucket: "{{ cloud_storage_telemetry_bucketname }}"
+ s3_endpoint: "{{ cloud_private_storage_endpoint }}"
+ s3_segment_dir: "druid/raw/segments"
+ s3_path_like_access: "{{ cloud_storage_pathstyle_access }}"
+ s3_v4_sign_region: "{{ cloud_private_storage_region }}"
+ #Logging the indexing logs to s3
+ s3_logging_bucket: "{{ cloud_storage_telemetry_bucketname }}"
+ s3_indexer_logs_dir: "druid/raw/stage/indexing_logs"
+ #Druid GCS Details
+ gcloud_bucket: "{{ cloud_storage_telemetry_bucketname }}"
+ gcloud_prefix: ""
+ gcloud_max_list_length: 1024
+ #Logging the indexing logs to GCS
+ gcloud_logging_bucket: "{{ cloud_storage_telemetry_bucketname }}"
+ gcloud_indexer_logs_dir: "druid/raw/stage/indexing_logs"
+ #Druid coordinator node configuration
+ druid_coordinator_heap_size: 128m
+ druid_coordinator_period: PT30S
+ druid_coordinator_startDelay: PT30S
+ druid_coordinator_balance_strategy: diskNormalized
+ #Druid overlord node configuration
+ druid_overlord_heap_size: 256m
+ #Druid broker node configuration
+ druid_broker_min_heap_size: 128m
+ druid_broker_max_heap_size: 128m
+ druid_broker_max_direct_size: 800m
+ druid_broker_http_numConnections: 5
+ druid_broker_server_http_numThread: 25
+ druid_broker_processing_bufferBytes: 134217728
+ druid_broker_processing_threads: 2
+ #Druid historical node configuration
+ druid_historical_min_heap_size: 1048m
+ druid_historical_max_heap_size: 1048m
+ druid_historical_max_direct_size: 800m
+ druid_historical_http_numConnections: 5
+ druid_historical_server_http_numThread: 25
+ druid_historical_processing_bufferBytes: 134217728
+ druid_historical_processing_threads: 2
+ druid_historical_enable_cache: false
+ druid_historical_segmentcache_size: 10000000000
+ druid_historical_server_maxsize: 10000000000
+ druid_historical_processing_num_merge_buffers: 2
+ druid_query_ondiskstorage_enabled: true
+ druid_historical_maxMergingDictionarySize: 100000000
+ druid_historical_segmentcache_numloadingthreads: 4
+ druid_historical_segmentcache_path: "/var/segmentstore"
+ druid.query.groupBy.maxOnDiskStorage: 10737418240
+ #Druid middlemanager configuration
+ druid_middlemanager_heap_size: 128m
+ druid_middlemanager_worker_cap: 4
+ druid_mm_java_opts_array: "-server -Xmx900m -XX:+UseG1GC -XX:HeapDumpPath={{ druid_mm_heap_dump_file }} -XX:MaxGCPauseMillis=100 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps"
+ druid_middlemanager_peon_server_http_numThread: 25
+ druid_middlemanager_peon_processing_bufferBytes: 25000000
+ druid_middlemanager_peon_processing_threads: 2
+ druid_middlemanager_peon_server_maxsize: 0
+ druid_indexing_queue_startDelay: PT30S
+ druid_router_heap_size: 1g
+ druid_router_http_numConnections: 50
+ druid_router_http_readTimeout: PT5M
+ druid_router_http_numMaxThreads: 100
+ druid_server_http_numThreads: 100
+ druid_router_managementProxy_enabled: true
+ druid_historical_maxOnDiskStorage: 10737418240
+ rollup:
+ #Druid Postgres Details
+ druid_postgres_db: "druid"
+ druid_postgres_host: "{{ postgres.db_url }}"
+ druid_postgres_port: "{{ postgres.db_port }}"
+ druid_postgres_user: "{{ druid_postgres_user | default('druid@' + postgres.db_url) }}"
+ #Druid Azure Details
+ druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}"
+ azure_account_name: "{{ cloud_private_storage_accountname }}"
+ azure_storage_secret: "{{ cloud_private_storage_secret }}"
+ azure_container: "{{ cloud_storage_telemetry_bucketname }}"
+ #Logging the indexing logs to azure
+ druid_log_azure_container: "{{ cloud_storage_telemetry_bucketname }}"
+ druid_log_azure_folder: "druidlogs"
+ #Druid S3 Details
+ druid_storage_type: "{{ cloud_storage_telemetry_type }}"
+ s3_access_key: "{{ s3_storage_key }}"
+ s3_secret_key: "{{ s3_storage_secret }}"
+ s3_bucket: "{{ s3_storage_container }}"
+ s3_endpoint: "{{ s3_storage_endpoint }}"
+ s3_segment_dir: "druid/rollup/segments"
+ s3_path_like_access: "{{ cloud_storage_pathstyle_access }}"
+ s3_v4_sign_region: "{{ cloud_private_storage_region }}"
+ #Logging the indexing logs to s3
+ s3_logging_bucket: "{{ cloud_storage_telemetry_bucketname }}"
+ s3_indexer_logs_dir: "druid/rollup/stage/indexing_logs"
+ #Druid GCS Details
+ druid.google.bucket: "{{ cloud_storage_telemetry_bucketname }}"
+ druid.google.prefix: ""
+ druid.google.maxListingLength: 1024
+ #Logging the indexing logs to gcs
+ gcloud_logging_bucket: "{{ cloud_storage_telemetry_bucketname }}"
+ gcloud_indexer_logs_dir: "druid/rollup/stage/indexing_logs"
+ #Druid coordinator node configuration
+ druid_coordinator_heap_size: 128m
+ druid_coordinator_period: PT30S
+ druid_coordinator_startDelay: PT30S
+ druid_coordinator_balance_strategy: cost
+ #Druid overlord node configuration
+ druid_overlord_heap_size: 256m
+ #Druid broker node configuration
+ druid_broker_min_heap_size: 128m
+ druid_broker_max_heap_size: 128m
+ druid_broker_max_direct_size: 700m
+ druid_broker_http_numConnections: 5
+ druid_broker_server_http_numThread: 25
+ druid_broker_processing_bufferBytes: 134217728
+ druid_broker_processing_threads: 2
+ #Druid historical node configuration
+ druid_historical_min_heap_size: 1048m
+ druid_historical_max_heap_size: 1048m
+ druid_historical_max_direct_size: 800m
+ druid_historical_http_numConnections: 5
+ druid_historical_server_http_numThread: 25
+ druid_historical_processing_bufferBytes: 134217728
+ druid_historical_processing_threads: 2
+ druid_historical_enable_cache: false
+ druid_historical_segmentcache_size: 2000000000
+ druid_historical_server_maxsize: 10000000000
+ druid_historical_processing_num_merge_buffers: 2
+ druid_query_ondiskstorage_enabled: false
+ druid_historical_segmentcache_numloadingthreads: 4
+ druid_historical_segmentcache_path: "/var/segmentstore"
+ #Druid middlemanager configuration
+ druid_middlemanager_heap_size: 128m
+ druid_middlemanager_worker_cap: 4
+ druid_mm_java_opts_array: "-server -Xmx900m -XX:+UseG1GC -XX:HeapDumpPath={{ druid_mm_heap_dump_file }} -XX:MaxGCPauseMillis=100 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps"
+ druid_middlemanager_peon_server_http_numThread: 25
+ druid_middlemanager_peon_processing_bufferBytes: 25000000
+ druid_middlemanager_peon_processing_threads: 2
+ druid_middlemanager_peon_server_maxsize: 0
+ druid_indexing_queue_startDelay: PT30S
+ druid_router_heap_size: 1g
+ druid_router_http_numConnections: 50
+ druid_router_http_readTimeout: PT5M
+ druid_router_http_numMaxThreads: 100
+ druid_server_http_numThreads: 100
+ druid_router_managementProxy_enabled: true
enable_druid_sql: true
diff --git a/ansible/roles/analytics-druid/templates/_common/common.runtime.properties b/ansible/roles/analytics-druid/templates/_common/common.runtime.properties
index 8a047b09ff..be217e65c8 100644
--- a/ansible/roles/analytics-druid/templates/_common/common.runtime.properties
+++ b/ansible/roles/analytics-druid/templates/_common/common.runtime.properties
@@ -70,6 +70,10 @@ druid.s3.endpoint.signingRegion={{ druid_configs[cluster].s3_v4_sign_region }}
# druid.storage.sse.type=s3
# uncomment to disable chunk encoding
# druid.s3.disableChunkedEncoding=true
+{% elif druid_storage_type == "google" %}
+druid.google.bucket={{ druid_configs[cluster].gcloud_bucket }}
+druid.google.prefix={{ druid_configs[cluster].gcloud_prefix }}
+druid.google.maxListingLength={{ druid_configs[cluster].gcloud_max_list_length }}
{% endif %}
# Indexing service logs
@@ -83,6 +87,9 @@ druid.indexer.logs.prefix= {{ druid_configs[cluster].druid_log_azure_folder }}
druid.indexer.logs.s3Bucket={{ druid_configs[cluster].s3_logging_bucket }}
# path to logs within the bucker
druid.indexer.logs.s3Prefix={{ druid_configs[cluster].s3_indexer_logs_dir }}
+{% elif druid_indexing_logs_type == "google" %}
+druid.indexer.logs.bucket={{ druid_configs[cluster].gcloud_logging_bucket }}
+druid.indexer.logs.prefix={{ druid_configs[cluster].gcloud_indexer_logs_dir }}
{% endif %}
# Service discovery
diff --git a/ansible/roles/analytics-spark-provision/defaults/main.yml b/ansible/roles/analytics-spark-provision/defaults/main.yml
index ef986228fe..211de20b11 100644
--- a/ansible/roles/analytics-spark-provision/defaults/main.yml
+++ b/ansible/roles/analytics-spark-provision/defaults/main.yml
@@ -3,7 +3,7 @@ analytics:
soft_path: /mount/data/analytics
base_path: /home/analytics
scala_version: 2.12.10
-spark_version: 3.1.3
+spark_version: 3.2.1
model_version: "2.0"
spark:
home: "{{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7"
diff --git a/ansible/roles/analytics-spark-provision/tasks/main.yml b/ansible/roles/analytics-spark-provision/tasks/main.yml
index 01eef6c9e1..65731cecb9 100644
--- a/ansible/roles/analytics-spark-provision/tasks/main.yml
+++ b/ansible/roles/analytics-spark-provision/tasks/main.yml
@@ -117,28 +117,31 @@
recurse: yes
become: yes
+# kenneth changed to install Ruby 2.6 as per R.4.6.0
- name: Install latest ruby
become: yes
become_user: "{{ analytics_user }}"
- shell: "export PATH=$PATH:/home/analytics/.rvm/bin && rvm install ruby-2.2"
+ shell: "export PATH=$PATH:/home/analytics/.rvm/bin && rvm install ruby-2.6"
- name: Add ruby repository
become: yes
apt_repository:
repo: ppa:brightbox/ruby-ng
+# kenneth changed to install ruby-dev 2.6 as per R.4.6.0 - there is no ruby2.2-dev in bionic
- name: Install latest ruby-dev
become: yes
apt:
- name: "ruby2.2-dev"
+ name: "ruby2.6-dev"
state: installed
update_cache: true
cache_valid_time: 3600
+# changed to ruby 2.6 as per R.4.6.0
- name: Install ruby-kafka
become: yes
become_user: "{{ analytics_user }}"
- shell: "bash -ilc 'export PATH=$PATH:/home/analytics/.rvm/bin && rvm --default use ruby-2.2 && gem install ruby-kafka'"
+ shell: "bash -ilc 'export PATH=$PATH:/home/analytics/.rvm/bin && rvm --default use ruby-2.6 && gem install --user-install --no-document ruby-kafka'"
- name: Download Kafka-2.11
become: yes
diff --git a/ansible/roles/analytics-spark-provision/templates/spark-env.j2 b/ansible/roles/analytics-spark-provision/templates/spark-env.j2
index dea6e5ad06..bb8a92abf0 100644
--- a/ansible/roles/analytics-spark-provision/templates/spark-env.j2
+++ b/ansible/roles/analytics-spark-provision/templates/spark-env.j2
@@ -70,12 +70,12 @@ export SPARK_WORKER_MEMORY={{ spark.worker.memory }}
export SPARK_WORKER_INSTANCES={{ spark.worker.instances }}
export SPARK_EXECUTOR_MEMORY={{ spark.executor.memory }}
export SPARK_PUBLIC_DNS="{{ spark.public_dns }}"
-export reports_storage_key={{sunbird_private_storage_account_name}}
-export reports_storage_secret={{sunbird_private_storage_account_key}}
-export azure_storage_key={{sunbird_private_storage_account_name}}
-export azure_storage_secret={{sunbird_private_storage_account_key}}
-export druid_storage_account_key={{sunbird_public_storage_account_name}}
-export druid_storage_account_secret={{sunbird_public_storage_account_key}}
-export aws_storage_key={{ s3_storage_key }}
-export aws_storage_secret={{ s3_storage_secret }}
+export reports_storage_key={{cloud_private_storage_accountname}}
+export reports_storage_secret={{cloud_private_storage_secret}}
+export azure_storage_key={{cloud_private_storage_accountname}}
+export azure_storage_secret={{cloud_private_storage_secret}}
+export druid_storage_account_key={{cloud_private_storage_accountname}}
+export druid_storage_account_secret={{cloud_private_storage_secret}}
+export aws_storage_key={{ cloud_private_storage_accountname }}
+export aws_storage_secret={{ cloud_private_storage_secret }}
diff --git a/ansible/roles/artifacts-download-azure/tasks/main.yml b/ansible/roles/artifacts-download-azure/tasks/main.yml
index db79bc213f..25fac02a27 100644
--- a/ansible/roles/artifacts-download-azure/tasks/main.yml
+++ b/ansible/roles/artifacts-download-azure/tasks/main.yml
@@ -1,8 +1,8 @@
---
- name: Ensure azure blob storage container exists
- command: az storage container exists --name {{ artifacts_container }}
+ command: az storage container exists --name {{ cloud_storage_artifacts_bucketname }}
- name: Download from azure blob storage
- command: az storage blob download -c {{ artifacts_container }} --name {{ artifact }} -f {{ artifact_path }}
+ command: az storage blob download -c {{ cloud_storage_artifacts_bucketname }} --name {{ artifact }} -f {{ artifact_path }}
async: 3600
poll: 10
diff --git a/ansible/roles/artifacts-upload-azure/tasks/main.yml b/ansible/roles/artifacts-upload-azure/tasks/main.yml
index 785dc1a455..91e66a5eec 100644
--- a/ansible/roles/artifacts-upload-azure/tasks/main.yml
+++ b/ansible/roles/artifacts-upload-azure/tasks/main.yml
@@ -1,8 +1,8 @@
---
- name: Ensure azure blob storage container exists
- command: az storage container create --name {{ artifacts_container }}
+ command: az storage container create --name {{ cloud_storage_artifacts_bucketname }}
- name: Upload to azure blob storage
- command: az storage blob upload -c {{ artifacts_container }} --name {{ artifact }} -f {{ artifact_path }}
+ command: az storage blob upload -c {{ cloud_storage_artifacts_bucketname }} --name {{ artifact }} -f {{ artifact_path }}
async: 3600
poll: 10
diff --git a/ansible/roles/azure-hdinsight-spark-cluster/templates/create-cluster.sh.j2 b/ansible/roles/azure-hdinsight-spark-cluster/templates/create-cluster.sh.j2
index 8a7703c936..76b4c45585 100644
--- a/ansible/roles/azure-hdinsight-spark-cluster/templates/create-cluster.sh.j2
+++ b/ansible/roles/azure-hdinsight-spark-cluster/templates/create-cluster.sh.j2
@@ -8,8 +8,8 @@ headnode_size="{{headnode_size}}"
location="{{location}}"
http_user=admin
http_password="{{azure_spark_cluster_http_password}}"
-storage_account_name="{{sunbird_private_storage_account_name}}"
-storage_account_key="{{sunbird_private_storage_account_key}}"
+storage_account_name="{{cloud_private_storage_accountname}}"
+storage_account_key="{{cloud_private_storage_secret}}"
storage_container="{{spark_storage_container}}"
subnet_name="{{subnet_name}}"
vnet_name="{{vnet_name}}"
diff --git a/ansible/roles/cassandra-backup/tasks/main.yml b/ansible/roles/cassandra-backup/tasks/main.yml
index f3cf8e916e..b4cc6dd8ed 100755
--- a/ansible/roles/cassandra-backup/tasks/main.yml
+++ b/ansible/roles/cassandra-backup/tasks/main.yml
@@ -56,6 +56,8 @@
name: gcp-cloud-storage
tasks_from: upload-batch.yml
vars:
+ gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+ gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
gcp_bucket_name: "{{ cloud_storage_dpcassandrabackup_bucketname }}"
gcp_path: "{{ cloud_storage_dpcassandrabackup_foldername }}/{{ cassandra_backup_gzip_file_name}}"
local_file_or_folder_path: "{{ cassandra_backup_gzip_file_path }}"
diff --git a/ansible/roles/content-snapshot-indexer/defaults/main.yml b/ansible/roles/content-snapshot-indexer/defaults/main.yml
index 40ba1ef8dd..436161e9c7 100644
--- a/ansible/roles/content-snapshot-indexer/defaults/main.yml
+++ b/ansible/roles/content-snapshot-indexer/defaults/main.yml
@@ -46,8 +46,8 @@ cloud_storage:
container: "telemetry-data-store" # Container is different in all env so override this.
object_key: "druid-content-snapshot/snapshot.txt"
provider: "azure"
- account_name: "{{sunbird_public_storage_account_name}}"
- account_key: "{{sunbird_public_storage_account_key}}"
+ account_name: "{{cloud_public_storage_accountname}}"
+ account_key: "{{cloud_public_storage_secret}}"
cassandra:
host: "{{lp_cassandra_host}}" ## LMS-Cassandra IP Address.
@@ -62,7 +62,7 @@ cassandra:
new_table: "device_profile_temp"
druid:
- coordinator_host: "{{ groups['rollup-coordinator'][0] | default(groups['raw-coordinator'][0]) }}"
+ coordinator_host: "{{ groups['rollup-coordinator'][0] | default(groups['raw-coordinator'][0]) }}"
data_source: "content-model-snapshot"
ingestion_spec_path: "{{ content_snapshot_jar_path }}/etl-jobs-1.0/druid_models/content_index_batch.json"
@@ -74,14 +74,21 @@ job_config:
cassandra_redis_path: "{{ content_snapshot_path }}/config/cassandraRedis.conf"
output_file_path: "{{ content_snapshot_path }}/compositeSearchBatchData"
-config_files: ["application","cassandraRedis","DeviceProfile","ESCloudUploader","ESContentIndexer","DialcodeRedisIndexer"]
-script_files: ["DeviceProfileScripts","DruidContentIndexer","RedisContentIndexer","RedisDialcodeIndexer","RedisUserDataIndexer","run-script"]
-
-
-
-
-
-
-
-
-
+config_files:
+ [
+ "application",
+ "cassandraRedis",
+ "DeviceProfile",
+ "ESCloudUploader",
+ "ESContentIndexer",
+ "DialcodeRedisIndexer",
+ ]
+script_files:
+ [
+ "DeviceProfileScripts",
+ "DruidContentIndexer",
+ "RedisContentIndexer",
+ "RedisDialcodeIndexer",
+ "RedisUserDataIndexer",
+ "run-script",
+ ]
diff --git a/ansible/roles/content-snapshot-indexer/templates/conf/DialcodeRedisIndexer.j2 b/ansible/roles/content-snapshot-indexer/templates/conf/DialcodeRedisIndexer.j2
index 337759f5d1..0aafd7cb74 100644
--- a/ansible/roles/content-snapshot-indexer/templates/conf/DialcodeRedisIndexer.j2
+++ b/ansible/roles/content-snapshot-indexer/templates/conf/DialcodeRedisIndexer.j2
@@ -10,7 +10,7 @@ redis.max.pipeline.size="{{ redis.max_pipeline_size }}"
redis.dialcode.database.index=6
-cloudStorage.accountName="{{sunbird_private_storage_account_name}}"
-cloudStorage.accountKey="{{sunbird_private_storage_account_key}}"
+cloudStorage.accountName="{{cloud_private_storage_accountname}}"
+cloudStorage.accountKey="{{cloud_private_storage_secret}}"
cloudStorage.container="{{ bucket | default('telemetry-data-store') }}"
cloudStorage.dialCodeDataFile="dialcode-data/dial_code.csv"
diff --git a/ansible/roles/content-snapshot-indexer/templates/conf/ESCloudUploader.j2 b/ansible/roles/content-snapshot-indexer/templates/conf/ESCloudUploader.j2
index 62312f203f..dab231923f 100755
--- a/ansible/roles/content-snapshot-indexer/templates/conf/ESCloudUploader.j2
+++ b/ansible/roles/content-snapshot-indexer/templates/conf/ESCloudUploader.j2
@@ -8,4 +8,5 @@ cloudStorage.container="{{ cloud_storage.container }}"
cloudStorage.objectKey="{{ cloud_storage.object_key }}"
cloudStorage.provider="{{ cloud_storage.provider }}"
cloudStorage.accountName="{{ cloud_storage.account_name }}"
-cloudStorage.accountKey="{{ cloud_storage.account_key }}"
\ No newline at end of file
+cloudStorage.accountKey="{{ cloud_storage.account_key }}"
+cloudStorage.accountEndpoint="{{ cloud_storage.account_endpoint }}"
\ No newline at end of file
diff --git a/ansible/roles/data-products-deploy/defaults/main.yml b/ansible/roles/data-products-deploy/defaults/main.yml
index 690c51d87d..06cdf1b0f2 100755
--- a/ansible/roles/data-products-deploy/defaults/main.yml
+++ b/ansible/roles/data-products-deploy/defaults/main.yml
@@ -1,6 +1,7 @@
analytics_user: analytics
analytics_group: analytics
spark_output_temp_dir: /mount/data/analytics/tmp/
+oci_install_loc: /home/{{analytics_user}}/bin/
bucket: "telemetry-data-store"
secor_bucket: "telemetry-data-store"
@@ -279,4 +280,15 @@ assessment_metric_primary_category: "{{ exhaust_job_assessment_primary_category
# Default s3 variables
sunbird_private_s3_storage_key: ""
-sunbird_private_s3_storage_secret: ""
\ No newline at end of file
+sunbird_private_s3_storage_secret: ""
+
+
+# jets3t s3 config, allows us to configure for s3-like object stores
+jets3t_s3_request_signature_version: "{{ s3_request_signature_version }}"
+jets3t_s3_endpoint_host: "{% if s3_storage_endpoint %}{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\\1') }}{% endif %}"
+jets3t_s3_disable_dns_buckets: "{{ s3_path_style_access }}"
+jets3t_s3_https_only: "{{ s3_https_only }}"
+jets3t_s3_default_bucket_location: "{{ s3_default_bucket_location }}"
+
+
+spark_pg_db_name: "analytics"
\ No newline at end of file
diff --git a/ansible/roles/data-products-deploy/tasks/main.yml b/ansible/roles/data-products-deploy/tasks/main.yml
index df495a5d4a..1c15e14023 100644
--- a/ansible/roles/data-products-deploy/tasks/main.yml
+++ b/ansible/roles/data-products-deploy/tasks/main.yml
@@ -5,6 +5,19 @@
tags:
- always
+- name: Ensure oci oss bucket exists
+ command: "{{oci_install_loc}}/oci os bucket get --name {{ bucket }}"
+ register: check_bucket
+ when: dp_object_store_type == "oci"
+ tags:
+ - always
+
+- name: Create oci oss bucket
+ command: "{{oci_install_loc}}/oci os bucket create -c {{oci_bucket_compartment}} --name {{bucket}}"
+ when: dp_object_store_type == "oci" and check_bucket.rc !=0
+ tags:
+ - always
+
- name: Copy Core Data Products
copy: src={{ analytics_batch_module_artifact }} dest={{ analytics.home }}/models-{{ model_version }}
tags:
@@ -14,6 +27,15 @@
command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }}
async: 3600
poll: 10
+ when: dp_object_store_type == "azure"
+ tags:
+ - dataproducts-spark-cluster
+
+- name: Copy Core Data Products to oci oss
+ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} --force"
+ async: 3600
+ poll: 10
+ when: dp_object_store_type == "oci"
tags:
- dataproducts-spark-cluster
@@ -27,8 +49,17 @@
command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar -f {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar
async: 3600
poll: 10
+ when: dp_object_store_type == "azure"
tags:
- - ed-dataproducts-spark-cluster
+ - ed-dataproducts-spark-cluster
+
+- name: Copy Ed Data Products to oci oss
+ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar --file {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar --force"
+ async: 3600
+ poll: 10
+ when: dp_object_store_type == "oci"
+ tags:
+ - ed-dataproducts-spark-cluster
- name: Copy Framework Library
copy: src={{ analytics_core_artifact }} dest={{ analytics.home }}/models-{{ model_version }}
@@ -39,6 +70,15 @@
command: az storage blob upload --overwrite --debug -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }}
async: 3600
poll: 10
+ when: dp_object_store_type == "azure"
+ tags:
+ - framework-spark-cluster
+
+- name: Copy Framework Library to oci oss
+ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} --force"
+ async: 3600
+ poll: 10
+ when: dp_object_store_type == "oci"
tags:
- framework-spark-cluster
@@ -51,6 +91,15 @@
command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }}
async: 3600
poll: 10
+ when: dp_object_store_type == "azure"
+ tags:
+ - framework-spark-cluster
+
+- name: Copy Scruid Library to oci oss
+ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} --force"
+ async: 3600
+ poll: 10
+ when: dp_object_store_type == "oci"
tags:
- framework-spark-cluster
@@ -87,6 +136,17 @@
line: 'spark_output_temp_dir="/var/log/sparkapp/tmp/"'
tags:
- framework-spark-cluster
+ when: dp_object_store_type != "oci"
+
+- name: Update spark temp dir value for cluster
+ lineinfile:
+ path: '{{ analytics.home }}/models-{{ model_version }}/application.conf'
+ regexp: '^spark_output_temp_dir="/mount/data/analytics/tmp/"'
+ line: 'spark_output_temp_dir="/var/log/spark/"'
+ tags:
+ - framework-spark-cluster
+ when: dp_object_store_type == "oci"
+
- name: Update logger kafka config for cluster
lineinfile:
@@ -100,9 +160,18 @@
command: az storage blob upload --overwrite -c {{ bucket }} -f {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf
async: 3600
poll: 10
+ when: dp_object_store_type == "azure"
tags:
- framework-spark-cluster
+- name: Copy configuration file to oci oss
+ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --file {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf --force"
+ async: 3600
+ poll: 10
+ when: dp_object_store_type == "oci"
+ tags:
+ - framework-spark-cluster
+
- name: Copy log4j2 xml file
template: src=log4j2.xml.j2 dest={{ analytics.home }}/models-{{ model_version }}/log4j2.xml mode=755 owner={{ analytics_user }} group={{ analytics_group }}
tags: [ dataproducts, framework, ed-dataproducts ]
@@ -271,7 +340,7 @@
- spark-jobs
- name: Copy collection-summary ingestion spec
- copy: src="collection-summary-ingestion-spec.json" dest={{ analytics.home }}/scripts/ mode=755 owner={{ analytics_user }} group={{ analytics_group }}
+ template: src="collection-summary-ingestion-spec.j2" dest={{ analytics.home }}/scripts/collection-summary-ingestion-spec.json mode=755 owner={{ analytics_user }} group={{ analytics_group }}
tags:
- ed-dataproducts
@@ -300,7 +369,7 @@
- spark-jobs
- name: Copy sourcing-summary ingestion spec
- copy: src="sourcing-ingestion-spec.json" dest={{ analytics.home }}/scripts/ mode=755 owner={{ analytics_user }} group={{ analytics_group }}
+ template: src="sourcing-ingestion-spec.j2" dest={{ analytics.home }}/scripts/sourcing-ingestion-spec.json mode=755 owner={{ analytics_user }} group={{ analytics_group }}
tags:
- ed-dataproducts
@@ -335,6 +404,24 @@
- run-job
- config-update
+- name: Copy jets3t.properties file
+ template: src=jets3t.j2 dest={{ analytics_cluster.home }}/jets3t.properties
+ delegate_to: localhost
+ tags:
+ - replay-job
+ - run-job
+ - config-update
+
+- name: Copy JetS3t.properties to oci oss
+ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/jets3t.properties --file {{ analytics_cluster.home }}/jets3t.properties --content-type auto --force"
+ async: 3600
+ poll: 10
+ when: dp_object_store_type == "oci"
+ tags:
+ - replay-job
+ - run-job
+ - config-update
+
- name: Replay Job
shell: "nohup {{ analytics_cluster.home }}/submit-script.sh --job {{ job_id }} --mode {{ mode }} --partitions {{ partitions }} --parallelisation {{ parallelisation }} --startDate {{ start_date }} --endDate {{ end_date }} --sparkMaster {{ sparkMaster }} --selectedPartitions {{ selected_partitions }} &"
async: "{{ (pause_min * 60) }}"
@@ -376,10 +463,10 @@
shell: |
if echo "{{jobs}}" | grep 'druid'
then
- python {{ analytics_cluster.home }}/update-job-requests.py {{ jobs }} {{ batch_size }} druid {{env}}_report_config
+ python {{ analytics_cluster.home }}/update-job-requests.py {{ jobs }} {{ batch_size }} druid {{spark_env}}_report_config
elif echo "{{jobs}}" | grep 'exhaust'
then
- python {{ analytics_cluster.home }}/update-job-requests.py {{ jobs }} {{ batch_size }} exhaust {{env}}_job_request
+ python {{ analytics_cluster.home }}/update-job-requests.py {{ jobs }} {{ batch_size }} exhaust {{spark_env}}_job_request
fi
tags:
- parallel-jobs-submit
diff --git a/ansible/roles/data-products-deploy/templates/cluster-config.json.j2 b/ansible/roles/data-products-deploy/templates/cluster-config.json.j2
index 1a26514684..25d6703c57 100644
--- a/ansible/roles/data-products-deploy/templates/cluster-config.json.j2
+++ b/ansible/roles/data-products-deploy/templates/cluster-config.json.j2
@@ -1,12 +1,37 @@
+
+{% if dp_object_store_type == "azure" %}
{
"jars": [
- "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_core_artifact }}",
- "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/{{ scruid_artifact }}",
- "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_ed_dataporducts_jar_artifact }}"
+ {% if cloud_storage_telemetry_type == "azure" %}
+ "wasbs://{{ bucket }}@{{cloud_private_storage_accountname}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_core_artifact }}",
+ "wasbs://{{ bucket }}@{{cloud_private_storage_accountname}}.blob.core.windows.net/models-{{ model_version }}/{{ scruid_artifact }}",
+ "wasbs://{{ bucket }}@{{cloud_private_storage_accountname}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_ed_dataporducts_jar_artifact }}"
+ {% elif cloud_storage_telemetry_type == "s3" %}
+ "s3://{{ bucket }}/models-{{ model_version }}/{{ analytics_core_artifact }}",
+ "s3://{{ bucket }}/models-{{ model_version }}/{{ scruid_artifact }}",
+ "s3://{{ bucket }}/models-{{ model_version }}/{{ analytics_ed_dataporducts_jar_artifact }}"
+ {% elif cloud_storage_telemetry_type == "gcloud" %}
+ "gs://{{ bucket }}/models-{{ model_version }}/{{ analytics_core_artifact }}",
+ "gs://{{ bucket }}/models-{{ model_version }}/{{ scruid_artifact }}",
+ "gs://{{ bucket }}/models-{{ model_version }}/{{ analytics_ed_dataporducts_jar_artifact }}"
+ {% endif %}
],
- "file": "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_batch_module_artifact }}",
+ "file":
+ {% if cloud_storage_telemetry_type == "azure" %}
+ "wasbs://{{ bucket }}@{{cloud_private_storage_accountname}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_batch_module_artifact }}",
+ {% elif cloud_storage_telemetry_type == "s3" %}
+ "s3://{{ bucket }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }}",
+ {% elif cloud_storage_telemetry_type == "gcloud" %}
+ "gs://{{ bucket }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }}",
+ {% endif %}
"files": [
- "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/application.conf"
+ {% if cloud_storage_telemetry_type == "azure" %}
+ "wasbs://{{ bucket }}@{{cloud_private_storage_accountname}}.blob.core.windows.net/models-{{ model_version }}/application.conf"
+ {% elif cloud_storage_telemetry_type == "s3" %}
+ "s3://{{ bucket }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }}/application.conf",
+ {% elif cloud_storage_telemetry_type == "gcloud" %}
+ "gs://{{ bucket }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }}/application.conf",
+ {% endif %}
],
"className": "org.ekstep.analytics.job.JobExecutor",
"executorCores": {{ spark_cluster.executor_core }},
@@ -21,7 +46,38 @@
"spark.cassandra.connection.timeoutMS" : "{{ spark_cassandra_connection_timeout_millis }}",
"spark.cassandra.read.timeoutMS" : "{{ spark_cassandra_query_timeout_millis }}",
"spark.cassandra.input.fetch.sizeInRows": "{{ spark_cassandra_query_max_rows_fetch_count }}",
- "spark.driver.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/sparkapp/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/hdp/current/spark2-client/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ sunbird_private_storage_account_name }} -Dazure_storage_secret={{ sunbird_private_storage_account_key }} -Dreports_storage_key={{sunbird_private_storage_account_name}} -Dreports_storage_secret={{sunbird_private_storage_account_key}} -Ddruid_storage_account_key={{ sunbird_public_storage_account_name }} -Ddruid_storage_account_secret={{sunbird_public_storage_account_key}}",
- "spark.executor.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/sparkapp/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/hdp/current/spark2-client/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ sunbird_private_storage_account_name }} -Dazure_storage_secret={{ sunbird_private_storage_account_key }} -Dreports_storage_key={{sunbird_private_storage_account_name}} -Dreports_storage_secret={{sunbird_private_storage_account_key}} -Ddruid_storage_account_key={{ sunbird_public_storage_account_name }} -Ddruid_storage_account_secret={{sunbird_public_storage_account_key}}"
+ "spark.driver.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/sparkapp/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/hdp/current/spark2-client/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ cloud_private_storage_accountname }} -Dazure_storage_secret={{ cloud_private_storage_secret }} -Dreports_storage_key={{cloud_private_storage_accountname}} -Dreports_storage_secret={{cloud_private_storage_secret}} -Ddruid_storage_account_key={{ cloud_public_storage_accountname }} -Ddruid_storage_account_secret={{cloud_public_storage_secret}}",
+ "spark.executor.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/sparkapp/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/hdp/current/spark2-client/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ cloud_private_storage_accountname }} -Dazure_storage_secret={{ cloud_private_storage_secret }} -Dreports_storage_key={{cloud_private_storage_accountname}} -Dreports_storage_secret={{cloud_private_storage_secret}} -Ddruid_storage_account_key={{ cloud_public_storage_accountname }} -Ddruid_storage_account_secret={{cloud_public_storage_secret}}"
}
}
+{% elif (dp_object_store_type == "oci") %}
+{
+ "jars": [
+ "oci://{{ bucket }}@{{ oci_namespace }}/models-{{ model_version }}/{{ analytics_core_artifact }}",
+ "oci://{{ bucket }}@{{ oci_namespace }}/models-{{ model_version }}/{{ scruid_artifact }}",
+ "oci://{{ bucket }}@{{ oci_namespace }}/models-{{ model_version }}/{{ analytics_ed_dataporducts_jar_artifact }}",
+ "oci://{{ bucket }}@{{ oci_namespace }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }}"
+ ],
+ "file": "oci://{{ bucket }}@{{ oci_namespace }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }}",
+ "files": [
+ "oci://{{ bucket }}@{{ oci_namespace }}/models-{{ model_version }}/application.conf",
+ "oci://{{ bucket }}@{{ oci_namespace }}/models-{{ model_version }}/jets3t.properties"
+ ],
+ "className": "org.ekstep.analytics.job.JobExecutor",
+ "executorCores": {{ spark_cluster.executor_core }},
+ "executorMemory": "{{ spark_cluster.executor_memory }}",
+ "numExecutors": {{ spark_cluster.num_executors }},
+ "conf": {
+ "spark.sql.autoBroadcastJoinThreshold" : "-1",
+ "spark.dynamicAllocation.enabled" :"{{ spark_enable_dynamic_allocation }}",
+ "spark.shuffle.service.enabled" :"{{ spark_enable_dynamic_allocation }}",
+ "spark.sql.shuffle.partitions" : "{{ spark_sql_shuffle_partitions }}",
+ "spark.scheduler.mode" : "FAIR",
+ "spark.cassandra.connection.timeoutMS" : "{{ spark_cassandra_connection_timeout_millis }}",
+ "spark.cassandra.read.timeoutMS" : "{{ spark_cassandra_query_timeout_millis }}",
+ "spark.cassandra.input.fetch.sizeInRows": "{{ spark_cassandra_query_max_rows_fetch_count }}",
+ "spark.driver.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/spark3/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/odh/2.0.1/spark/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ cloud_private_storage_accountname }} -Dazure_storage_secret={{ cloud_private_storage_secret }} -Dreports_storage_key={{cloud_private_storage_accountname}} -Dreports_storage_secret={{cloud_private_storage_secret}} -Ddruid_storage_account_key={{ cloud_public_storage_accountname }} -Ddruid_storage_account_secret={{cloud_public_storage_secret}}, -Daws_storage_key={{ cloud_private_storage_accountname }} -Daws_storage_secret={{ cloud_private_storage_secret }} -Dcom.amazonaws.sdk.s3.defaultStreamBufferSize=512m",
+ "spark.executor.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/spark3/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/odh/2.0.1/spark/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ cloud_private_storage_accountname }} -Dazure_storage_secret={{ cloud_private_storage_secret }} -Dreports_storage_key={{cloud_private_storage_accountname}} -Dreports_storage_secret={{cloud_private_storage_secret}} -Ddruid_storage_account_key={{ cloud_public_storage_accountname }} -Ddruid_storage_account_secret={{cloud_public_storage_secret}} -Daws_storage_key={{ cloud_private_storage_accountname }} -Daws_storage_secret={{ cloud_private_storage_secret }} -Dcom.amazonaws.sdk.s3.defaultStreamBufferSize=512m"
+ }
+}
+{% endif %}
\ No newline at end of file
diff --git a/ansible/roles/data-products-deploy/templates/collection-summary-ingestion-spec.j2 b/ansible/roles/data-products-deploy/templates/collection-summary-ingestion-spec.j2
new file mode 100644
index 0000000000..f26c2e6447
--- /dev/null
+++ b/ansible/roles/data-products-deploy/templates/collection-summary-ingestion-spec.j2
@@ -0,0 +1,256 @@
+{
+ "type": "index",
+ "spec": {
+ "dataSchema": {
+ "dataSource": "collection-summary-snapshot",
+ "parser": {
+ "type": "string",
+ "parseSpec": {
+ "format": "json",
+ "flattenSpec": {
+ "useFieldDiscovery": false,
+ "fields": [
+ {
+ "type": "root",
+ "name": "content_org",
+ "expr": "contentorg"
+ },
+ {
+ "type": "root",
+ "name": "user_org",
+ "expr": "orgname"
+ },
+ {
+ "type": "root",
+ "name": "batch_start_date",
+ "expr": "startdate"
+ },
+ {
+ "type": "root",
+ "name": "batch_end_date",
+ "expr": "enddate"
+ },
+ {
+ "type": "root",
+ "name": "has_certificate",
+ "expr": "hascertified"
+ },
+ {
+ "type": "root",
+ "name": "collection_id",
+ "expr": "courseid"
+ },
+ {
+ "type": "root",
+ "name": "batch_id",
+ "expr": "batchid"
+ },
+ {
+ "type": "root",
+ "name": "collection_name",
+ "expr": "collectionname"
+ },
+ {
+ "type": "root",
+ "name": "batch_name",
+ "expr": "batchname"
+ },
+ {
+ "type": "root",
+ "name": "total_enrolment",
+ "expr": "enrolleduserscount"
+ },
+ {
+ "type": "root",
+ "name": "total_completion",
+ "expr": "completionuserscount"
+ },
+ {
+ "type": "root",
+ "name": "total_certificates_issued",
+ "expr": "certificateissuedcount"
+ },
+ {
+ "type": "root",
+ "name": "content_status",
+ "expr": "contentstatus"
+ },
+ {
+ "type": "root",
+ "name": "user_state",
+ "expr": "state"
+ },
+ {
+ "type": "root",
+ "name": "user_district",
+ "expr": "district"
+ },
+ {
+ "type": "root",
+ "name": "content_channel",
+ "expr": "channel"
+ },
+ {
+ "type": "root",
+ "name": "keywords",
+ "expr": "keywords"
+ },
+ {
+ "type": "root",
+ "name": "timestamp",
+ "expr": "timestamp"
+ },
+ {
+ "type": "root",
+ "name": "medium",
+ "expr": "medium"
+ },
+ {
+ "type": "root",
+ "name": "subject",
+ "expr": "subject"
+ },
+ {
+ "type": "root",
+ "name": "created_for",
+ "expr": "createdfor"
+ },
+ {
+ "type": "root",
+ "name": "user_type",
+ "expr": "usertype"
+ },
+ {
+ "type": "root",
+ "name": "user_subtype",
+ "expr": "usersubtype"
+ }
+ ]
+ },
+ "dimensionsSpec": {
+ "dimensions": [
+ {
+ "name": "content_org"
+ },
+ {
+ "name": "user_org"
+ },
+ {
+ "type": "string",
+ "name": "batch_id"
+ },
+ {
+ "type": "string",
+ "name": "batch_start_date"
+ },
+ {
+ "type": "string",
+ "name": "batch_end_date"
+ },
+ {
+ "type": "string",
+ "name": "collection_id"
+ },
+ {
+ "type": "string",
+ "name": "collection_name"
+ },
+ {
+ "type": "string",
+ "name": "batch_name"
+ },
+ {
+ "type": "long",
+ "name": "total_enrolment"
+ },
+ {
+ "type": "long",
+ "name": "total_completion"
+ },
+ {
+ "type": "long",
+ "name": "total_certificates_issued"
+ },
+ {
+ "type": "string",
+ "name": "content_status"
+ },
+ {
+ "type": "string",
+ "name": "user_state"
+ },
+ {
+ "type": "string",
+ "name": "user_district"
+ },
+ {
+ "name": "keywords"
+ },
+ {
+ "name": "has_certificate"
+ },
+ {
+ "type": "string",
+ "name": "content_channel"
+ },
+ {
+ "name": "medium"
+ },
+ {
+ "name": "subject"
+ },
+ {
+ "name": "created_for"
+ },
+ {
+ "type": "string",
+ "name": "user_type"
+ },
+ {
+ "type": "string",
+ "name": "user_subtype"
+ }
+ ],
+ "dimensionsExclusions": []
+ },
+ "timestampSpec": {
+ "column": "timestamp",
+ "format": "auto"
+ }
+ }
+ },
+ "metricsSpec": [],
+ "granularitySpec": {
+ "type": "uniform",
+ "segmentGranularity": "day",
+ "queryGranularity": "none",
+ "rollup": true
+ }
+ },
+ "ioConfig": {
+ "type": "index",
+ "firehose": {
+{% if dp_object_store_type == "azure" %}
+ "type": "static-azure-blobstore",
+ "blobs": [
+ {
+ "container": "{{reports_container}}",
+ "path": "/collection-summary-reports-v2/collection-summary-report-latest.json"
+ }
+ ],
+{% elif (dp_object_store_type == "oci") %}
+ "type": "static-s3",
+ "uris": [ "s3://{{reports_container}}/sourcing/SourcingSummaryReport.json"],
+{% endif %}
+ "fetchTimeout": 300000
+ }
+ },
+ "tuningConfig": {
+ "type": "index",
+ "targetPartitionSize": 5000000,
+ "maxRowsInMemory": 25000,
+ "forceExtendableShardSpecs": false,
+ "logParseExceptions": true
+ }
+ }
+}
\ No newline at end of file
diff --git a/ansible/roles/data-products-deploy/templates/common.conf.j2 b/ansible/roles/data-products-deploy/templates/common.conf.j2
index bde88ec9d4..d29f854f8f 100644
--- a/ansible/roles/data-products-deploy/templates/common.conf.j2
+++ b/ansible/roles/data-products-deploy/templates/common.conf.j2
@@ -8,17 +8,15 @@ service.search.path="{{ service.search.path }}"
spark.cassandra.connection.host="{{groups['dp-cassandra'][0]}}"
cassandra.keyspace_prefix="{{ cassandra_keyspace_prefix }}"
cassandra.hierarchy_store_prefix="{{ cassandra_hierarchy_store_prefix }}"
-
-
storage.key.config="{{ dp_storage_key_config }}"
storage.secret.config="{{ dp_storage_secret_config }}"
reports.storage.key.config="{{ dp_reports_storage_key_config }}"
reports.storage.secret.config="{{ dp_reports_storage_secret_config }}"
{% if dp_object_store_type == "azure" %}
-cloud_storage_type="azure"
+cloud_storage_telemetry_type="azure"
{% elif (dp_object_store_type == "cephs3" or dp_object_store_type == "s3") %}
-cloud_storage_type="s3"
-cloud_storage_endpoint="{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\\1') }}"
+cloud_storage_telemetry_type="s3"
+cloud_private_storage_endpoint="{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\\1') }}"
cloud_storage_endpoint_with_protocol="{{ s3_storage_endpoint }}"
aws_storage_key="{{ s3_storage_key }}"
aws_storage_secret="{{ s3_storage_secret }}"
@@ -112,7 +110,7 @@ azure {
}
## Reports - Global config
-cloud.container.reports="reports"
+cloud.container.reports="{{cloud_storage_privatereports_bucketname}}"
# course metrics container in azure
course.metrics.cassandra.sunbirdKeyspace="sunbird"
@@ -218,7 +216,7 @@ metric.kafka.broker="{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
metric.kafka.topic="{{ env }}.prom.monitoring.metrics"
//Postgres Config
-postgres.db="{{postgres.db_name}}"
+postgres.db="{{postgres.spark_db_name}}"
postgres.url="jdbc:postgresql://{{postgres.db_url}}:{{postgres.db_port}}/"
postgres.user="{{postgres.db_username}}"
postgres.pass="{{postgres.db_password}}"
@@ -251,10 +249,14 @@ dcetextbook.filename="DCE_textbook_data.csv"
etbtextbook.filename="ETB_textbook_data.csv"
etb.dialcode.druid.length={{ etb_dialcode_list_druid_length }}
-
+{% if dp_object_store_type == "azure" %}
druid.report.default.storage="azure"
+{% elif (dp_object_store_type == "oci" or dp_object_store_type == "s3") %}
+druid.report.default.storage="s3"
+{% endif %}
+
druid.report.date.format="yyyy-MM-dd"
-druid.report.default.container="report-verification"
+druid.report.default.container="{{cloud_storage_report_verfication_bucketname}}"
## Collection Exhaust Jobs Configuration -- Start ##
@@ -305,4 +307,4 @@ uci.fushionauth.postgres.pass="{{ uci_postgres.fushionauth_db_psss }}"
uci.exhaust.store.prefix=""
uci.encryption.secret="{{ uci_encryption_secret_key }}"
-// END OF UCI Related Job Configs
\ No newline at end of file
+// END OF UCI Related Job Configs
diff --git a/ansible/roles/data-products-deploy/templates/jets3t.j2 b/ansible/roles/data-products-deploy/templates/jets3t.j2
new file mode 100644
index 0000000000..1ca346578c
--- /dev/null
+++ b/ansible/roles/data-products-deploy/templates/jets3t.j2
@@ -0,0 +1,8 @@
+storage-service.request-signature-version={{ jets3t_s3_request_signature_version }}
+s3service.disable-dns-buckets={{ jets3t_s3_disable_dns_buckets }}
+s3service.https-only=true
+{% if jets3t_s3_default_bucket_location %}
+s3service.default-bucket-location={{ jets3t_s3_default_bucket_location }}
+{% endif %}
+uploads.stream-retry-buffer-size=2147483646
+s3service.s3-endpoint={% if jets3t_s3_endpoint_host %}{{ jets3t_s3_endpoint_host }}{% else %}s3-ap-south-1.amazonaws.com{% endif %}
diff --git a/ansible/roles/data-products-deploy/templates/model-config.j2 b/ansible/roles/data-products-deploy/templates/model-config.j2
index 2bb0a042ea..f9991e1714 100644
--- a/ansible/roles/data-products-deploy/templates/model-config.j2
+++ b/ansible/roles/data-products-deploy/templates/model-config.j2
@@ -30,19 +30,24 @@ config() {
if [ ! -z "$2" ]; then keyword=$2; fi
case "$1" in
"assessment-correction")
- echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"unique/raw/","endDate":"'$endDate'","delta":0}]},"model":"org.sunbird.analytics.model.report.AssessmentCorrectionModel","modelParams":{"parallelization":200,"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all","aggregations":[{"name":"count","type":"count","fieldName":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"}],"filters":[{"type":"equals","dimension":"contentType","value":"SelfAssess"}],"descending":"false"},"fileOutputConfig":{"to":"file","params":{"file":"{{ analytics.home }}/assessment-correction/skippedEvents"}},"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'"},"output":[{"to":"kafka","params":{"brokerList":"'$brokerIngestionList'","topic":"'$assessTopic'"}}],"parallelization":200,"appName":"Assessment Correction Model"}'
+ echo '{"search":{"type":"{{dp_object_store_type}}","queries":[{"bucket":"'$bucket'","prefix":"unique/raw/","endDate":"'$endDate'","delta":0}]},"model":"org.sunbird.analytics.model.report.AssessmentCorrectionModel","modelParams":{"parallelization":200,"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all","aggregations":[{"name":"count","type":"count","fieldName":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"}],"filters":[{"type":"equals","dimension":"contentType","value":"SelfAssess"}],"descending":"false"},"fileOutputConfig":{"to":"file","params":{"file":"{{ analytics.home }}/assessment-correction/skippedEvents"}},"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'"},"output":[{"to":"kafka","params":{"brokerList":"'$brokerIngestionList'","topic":"'$assessTopic'"}}],"parallelization":200,"appName":"Assessment Correction Model"}'
;;
"assessment-archival")
- echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"store":"azure","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Job"}'
+ echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"store":"{{ dp_object_store_type }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Job"}'
;;
"assessment-archived-removal")
+{% if dp_object_store_type == "azure" %}
echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"deleteArchivedBatch":true,"azureFetcherConfig":{"store":"azure","blobExt":"csv.gz","reportPath":"archived-data/","container":"reports"},"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Removal Job"}'
;;
+{% elif (dp_object_store_type == "oci" or dp_object_store_type == "s3") %}
+ echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"deleteArchivedBatch":true,"azureFetcherConfig":{"store":"{{dp_object_store_type}}","blobExt":"csv.gz","reportPath":"archived-data/","container":"{{reports_container}}"},"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Removal Job"}'
+ ;;
+{% endif %}
"collection-reconciliation-job")
echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.CollectionReconciliationJob","modelParams":{"mode":"prodrun","brokerList":"{{ingestion_kafka_broker_host}}","topic":"{{env}}.issue.certificate.request","sparkCassandraConnectionHost":"{{ core_cassandra_host }}"},"parallelization":30,"appName":"CollectionReconciliationJob"}'
;;
"collection-summary-report")
- echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"], "contentType": "Course"},"fields":["identifier","name","organisation","channel"],"limit":10000}},"store":"azure","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}'
+ echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"],"contentType":"Course"},"fields":["identifier","name","organisation","channel"],"limit":10000}},"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}'
;;
"score-metric-migration-job")
echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.ScoreMetricMigrationJob","modelParams":{"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Score Metric Migration Job"}'
@@ -51,34 +56,34 @@ config() {
echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.AssessmentScoreCorrectionJob","modelParams":{"assessment.score.correction.batches":"","cassandraReadConsistency":"QUORUM","cassandraWriteConsistency":"QUORUM","csvPath":"/mount/data/analytics/score_correction","isDryRunMode":true,"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":30,"appName":"Assessment Score Correction Job"}'
;;
"course-batch-status-updater")
- echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.CourseBatchStatusUpdaterJob","modelParams":{"store":"azure","sparkElasticsearchConnectionHost":"http://{{ single_node_es_host }}:9200","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","kpLearningBasePath":"http://{{groups['learning'][0]}}:8080/learning-service","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Course Batch Status Updater Job"}'
+ echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.CourseBatchStatusUpdaterJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkElasticsearchConnectionHost":"http://{{ single_node_es_host }}:9200","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","kpLearningBasePath":"http://{{groups['learning'][0]}}:8080/learning-service","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Course Batch Status Updater Job"}'
;;
"collection-summary-report-v2")
- echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJobV2","modelParams":{"storageKeyConfig":"druid_storage_account_key","storageSecretConfig":"druid_storage_account_secret","batchSize":50,"generateForAllBatches":true,"contentFields":["identifier","name","organisation","channel","status","keywords","createdFor","medium","subject"],"contentStatus":["Live","Unlisted","Retired"],"store":"azure","specPath":"/mount/data/analytics/scripts/collection-summary-ingestion-spec.json","druidIngestionUrl":"'$druidIngestionURL'","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report V2"}'
+ echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJobV2","modelParams":{"storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","batchSize":50,"generateForAllBatches":true,"contentFields":["identifier","name","organisation","channel","status","keywords","createdFor","medium","subject"],"contentStatus":["Live","Unlisted","Retired"],"store":"{{dp_object_store_type}}","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","specPath":"/mount/data/analytics/scripts/collection-summary-ingestion-spec.json","druidIngestionUrl":"'$druidIngestionURL'","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report V2"}'
;;
"uci-private-exhaust")
- echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.uci.UCIPrivateExhaustJob","modelParams":{"store":"azure","mode":"OnDemand","storageContainer":"reports","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UCI Private Exhaust"}'
+ echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.uci.UCIPrivateExhaustJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UCI Private Exhaust"}'
;;
"uci-response-exhaust")
- echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"unique/raw/","endDate":"'$endDate'","delta":0}]},"filters":[{"name":"eid","operator":"EQ","value":"ASSESS"}],"model":"org.sunbird.analytics.uci.UCIResponseExhaust","modelParams":{"store":"azure","botPdataId":"{{ uci_pdata_id }}","mode":"OnDemand","fromDate":"","toDate":"","storageContainer":"reports"},"parallelization":8,"appName":"UCI Response Exhaust"}'
+ echo '{"search":{"type":"{{dp_object_store_type}}","queries":[{"bucket":"'$bucket'","prefix":"unique/raw/","endDate":"'$endDate'","delta":0}]},"filters":[{"name":"eid","operator":"EQ","value":"ASSESS"}],"model":"org.sunbird.analytics.uci.UCIResponseExhaust","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","botPdataId":"{{ uci_pdata_id }}","mode":"OnDemand","fromDate":"","toDate":""},"parallelization":8,"appName":"UCI Response Exhaust"}'
;;
"userinfo-exhaust")
- echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.UserInfoExhaustJob","modelParams":{"store":"azure","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UserInfo Exhaust"}'
+ echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.UserInfoExhaustJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UserInfo Exhaust"}'
;;
"program-collection-summary-report")
- echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"],"contentType":"Course","keywords":["'$keyword'"]},"fields":["identifier","name","organisation","channel"],"limit":10000}},"columns":["Published by","Batch id","Collection id","Collection name","Batch start date","Batch end date","State","Total enrolments By State","Total completion By State"], "keywords":"'$keyword'", "store":"azure","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}'
+ echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"],"contentType":"Course","keywords":["'$keyword'"]},"fields":["identifier","name","organisation","channel"],"limit":10000}},"columns":["Published by","Batch id","Collection id","Collection name","Batch start date","Batch end date","State","Total enrolments By State","Total completion By State"],"keywords":"'$keyword'","store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}'
;;
"response-exhaust")
- echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJob","modelParams":{"store":"azure","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust"}'
+ echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust"}'
;;
"response-exhaust-v2")
- echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2","modelParams":{"store":"azure","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust V2"}'
+ echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust V2"}'
;;
"progress-exhaust")
- echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ProgressExhaustJob","modelParams":{"store":"azure","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust"}'
+ echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ProgressExhaustJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}", "apiVersion":"v2", "parallelization":200,"mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust"}'
;;
"progress-exhaust-v2")
- echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ProgressExhaustJobV2","modelParams":{"store":"azure","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust V2"}'
+ echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ProgressExhaustJobV2","modelParams":{"store":"{{dp_object_store_type}}","mode":"OnDemand","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust V2"}'
;;
"druid_reports")
echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.DruidQueryProcessingModel","modelParams":{"mode":"batch"},"parallelization":8,"appName":"Druid Reports"}'
@@ -93,23 +98,23 @@ config() {
echo '{"jobsCount":'$jobManagerJobsCount',"topic":"'$job_topic'","bootStrapServer":"'$brokerList'","zookeeperConnect":"'$zookeeper'","consumerGroup":"jobmanager","slackChannel":"#test_channel","slackUserName":"JobManager","tempBucket":"'$bucket'","tempFolder":"'$temp_folder'"}'
;;
"wfs")
- echo '{"search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"{{ dp_raw_telemetry_backup_location }}","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"storageKeyConfig":"{{ dp_storage_key_config }}", "storageSecretConfig":"{{ dp_storage_secret_config }}", "apiVersion":"v2", "parallelization":200},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"} }],"parallelization":200,"appName":"Workflow Summarizer","deviceMapping":true}'
- #echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"unique/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"apiVersion":"v2"},"output":[{"to":"console","params":{"printEvent": false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":8,"appName":"Workflow Summarizer","deviceMapping":true}'
+ echo '{"search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"{{ dp_raw_telemetry_backup_location }}","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","store":"{{ dp_object_store_type }}","apiVersion":"v2","parallelization":200},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":200,"appName":"Workflow Summarizer","deviceMapping":true}'
+ #echo '{"search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"unique/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"apiVersion":"v2"},"output":[{"to":"console","params":{"printEvent": false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":8,"appName":"Workflow Summarizer","deviceMapping":true}'
;;
"video-streaming")
- echo '{"search":{"type":"azure"},"model":"org.ekstep.analytics.job.VideoStreamingJob","modelParams":{"maxIterations":10},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Video Streaming Job","deviceMapping":false}'
+ echo '{"search":{"type":"{{ dp_object_store_type }}"},"model":"org.ekstep.analytics.job.VideoStreamingJob","modelParams":{"maxIterations":10},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Video Streaming Job","deviceMapping":false}'
;;
"admin-user-reports")
- echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminReportJob","modelParams":{"fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin User Reports","deviceMapping":false}'
+ echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminReportJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin User Reports","deviceMapping":false}'
;;
"admin-geo-reports")
- echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminGeoReportJob","modelParams":{"fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin Geo Reports","deviceMapping":false}'
+ echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminGeoReportJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin Geo Reports","deviceMapping":false}'
;;
"telemetry-replay")
- echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"'$inputBucket'","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.job.EventsReplayJob","modelParams":{},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$sinkTopic'"}}],"parallelization":8,"appName":"TelemetryReplayJob","deviceMapping":false}'
+ echo '{"search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"'$inputBucket'","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.job.EventsReplayJob","modelParams":{},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$sinkTopic'"}}],"parallelization":8,"appName":"TelemetryReplayJob","deviceMapping":false}'
;;
"summary-replay")
- echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"derived/wfs/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.job.EventsReplayJob","modelParams":{},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$sinkTopic'"}}],"parallelization":8,"appName":"SummaryReplayJob","deviceMapping":false}'
+ echo '{"search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"derived/wfs/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.job.EventsReplayJob","modelParams":{},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$sinkTopic'"}}],"parallelization":8,"appName":"SummaryReplayJob","deviceMapping":false}'
;;
"content-rating-updater")
echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.updater.UpdateContentRating","modelParams": {"startDate": "'$endDate'","endDate": "'$endDate'"},"output": [{"to":"console","params":{"printEvent":false}}],"parallelization": 8,"appName": "Content Rating Updater","deviceMapping": false}'
@@ -118,25 +123,25 @@ config() {
echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.ExperimentDefinitionModel","modelParams":{"sparkElasticsearchConnectionHost":"{{ lp_composite_search_host }}"},"output":[{"to":"elasticsearch","params":{"index":"experiment"}}],"parallelization":8,"appName":"Experiment-Definition","deviceMapping":false}'
;;
"etb-metrics")
- echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.ETBMetricsJob","modelParams":{"reportConfig":{"id":"etb_metrics","metrics":[],"labels":{"date":"Date","identifier":"Textbook ID","name":"Textbook Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","createdOn":"Created On","lastUpdatedOn":"Last Updated On","totalQRCodes":"Total number of QR codes","contentLinkedQR":"Number of QR codes with atleast 1 linked content","withoutContentQR":"Number of QR codes with no linked content","withoutContentT1":"Term 1 QR Codes with no linked content","withoutContentT2":"Term 2 QR Codes with no linked content","status":"Textbook Status","totalContentLinked":"Total content linked","totalQRLinked":"Total QR codes linked to content","totalQRNotLinked":"Total number of QR codes with no linked content","leafNodesCount":"Total number of leaf nodes","leafNodeUnlinked":"Number of leaf nodes with no content","l1Name":"Level 1 Name","l2Name":"Level 2 Name","l3Name":"Level 3 Name","l4Name":"Level 4 Name","l5Name":"Level 5 Name","dialcode":"QR Code","sum(scans)":"Total Scans","noOfContent":"Number of contents","nodeType":"Type of Node","term":"Term"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"dialcode_counts.csv","postContainer":"'$reportPostContainer'"}},"dialcodeReportConfig":{"id":"etb_metrics","metrics":[],"labels":{},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":1,"reportPath":"dialcode_counts.csv","rollupAge":"ACADEMIC_YEAR","rollupCol":"Date","rollupRange":10,"postContainer":"'$reportPostContainer'"}},"etbFileConfig":{"bucket":"'$reportPostContainer'","file":"dialcode_scans/dialcode_counts.csv"},"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"channel","aliasName":"channel"},{"fieldName":"identifier","aliasName":"identifier","type":"Extraction","outputType":"STRING","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.split(\".\")[0]}"}]},{"fieldName":"name","aliasName":"name"},{"fieldName":"status","aliasName":"status"}],"filters":[{"type":"equals","dimension":"contentType","value":"TextBook"},{"type":"in","dimension":"status","values":["Live","Draft","Review"]}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"tenantConfig":{"tenantId":"","slugName":""},"store":"azure","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"ETB Metrics Model","deviceMapping":false}'
+ echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.ETBMetricsJob","modelParams":{"reportConfig":{"id":"etb_metrics","metrics":[],"labels":{"date":"Date","identifier":"Textbook ID","name":"Textbook Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","createdOn":"Created On","lastUpdatedOn":"Last Updated On","totalQRCodes":"Total number of QR codes","contentLinkedQR":"Number of QR codes with atleast 1 linked content","withoutContentQR":"Number of QR codes with no linked content","withoutContentT1":"Term 1 QR Codes with no linked content","withoutContentT2":"Term 2 QR Codes with no linked content","status":"Textbook Status","totalContentLinked":"Total content linked","totalQRLinked":"Total QR codes linked to content","totalQRNotLinked":"Total number of QR codes with no linked content","leafNodesCount":"Total number of leaf nodes","leafNodeUnlinked":"Number of leaf nodes with no content","l1Name":"Level 1 Name","l2Name":"Level 2 Name","l3Name":"Level 3 Name","l4Name":"Level 4 Name","l5Name":"Level 5 Name","dialcode":"QR Code","sum(scans)":"Total Scans","noOfContent":"Number of contents","nodeType":"Type of Node","term":"Term"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"dialcode_counts.csv","postContainer":"'$reportPostContainer'"}},"dialcodeReportConfig":{"id":"etb_metrics","metrics":[],"labels":{},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":1,"reportPath":"dialcode_counts.csv","rollupAge":"ACADEMIC_YEAR","rollupCol":"Date","rollupRange":10,"postContainer":"'$reportPostContainer'"}},"etbFileConfig":{"bucket":"'$reportPostContainer'","file":"dialcode_scans/dialcode_counts.csv"},"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"channel","aliasName":"channel"},{"fieldName":"identifier","aliasName":"identifier","type":"Extraction","outputType":"STRING","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.split(\".\")[0]}"}]},{"fieldName":"name","aliasName":"name"},{"fieldName":"status","aliasName":"status"}],"filters":[{"type":"equals","dimension":"contentType","value":"TextBook"},{"type":"in","dimension":"status","values":["Live","Draft","Review"]}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"tenantConfig":{"tenantId":"","slugName":""},"store":"{{ dp_object_store_type }}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"ETB Metrics Model","deviceMapping":false}'
;;
"course-enrollment-report")
- echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.CourseEnrollmentJob","modelParams":{"reportConfig":{"id":"tpd_metrics","metrics":[],"labels":{"completionCount":"Completion Count","status":"Status","enrollmentCount":"Enrollment Count","courseName":"Course Name","batchName":"Batch Name"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"DAY","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"course_enrollment.csv"}},"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]},"limit":10000}},"store":"azure","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Enrollment Metrics Model","deviceMapping":false}'
+ echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.CourseEnrollmentJob","modelParams":{"reportConfig":{"id":"tpd_metrics","metrics":[],"labels":{"completionCount":"Completion Count","status":"Status","enrollmentCount":"Enrollment Count","courseName":"Course Name","batchName":"Batch Name"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"DAY","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"course_enrollment.csv"}},"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]},"limit":10000}},"store":"{{ dp_object_store_type }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Enrollment Metrics Model","deviceMapping":false}'
;;
"course-consumption-report")
- echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.model.report.CourseConsumptionJob","modelParams": {"esConfig": {"request": {"filters": {"objectType": ["Content"],"contentType": ["Course"],"identifier": [],"status": ["Live"]}}},"reportConfig": {"id": "tpd_metrics","labels": {"date": "Date","status": "Batch Status","timespent": "Timespent in mins","courseName": "Course Name","batchName": "Batch Name"},"dateRange": {"staticInterval": "LastDay","granularity": "all"},"metrics": [{"metric": "totalCoursePlays","label": "Total Course Plays (in mins)","druidQuery": {"queryType": "groupBy","dataSource": "summary-events","intervals":"LastDay","aggregations": [{"name": "sum__edata_time_spent","type": "doubleSum","fieldName": "edata_time_spent"}],"dimensions": [{"fieldName": "object_rollup_l1","aliasName": "courseId"}, {"fieldName": "uid","aliasName": "userId"}, {"fieldName": "context_cdata_id","aliasName": "batchId"}],"filters": [{"type": "equals","dimension": "eid","value": "ME_WORKFLOW_SUMMARY"}, {"type": "in","dimension": "dimensions_pdata_id","values": ["'$producerEnv'.app", "'$producerEnv'.portal"]}, {"type": "equals","dimension": "dimensions_type","value": "content"}, {"type": "equals","dimension": "dimensions_mode","value": "play"}, {"type": "equals","dimension": "context_cdata_type","value": "batch"}],"postAggregation": [{"type": "arithmetic","name": "timespent","fields": {"leftField": "sum__edata_time_spent","rightField": 60,"rightFieldType": "constant"},"fn": "/"}],"descending": "false"}}],"output": [{"type": "csv","metrics": ["timespent"],"dims": []}],"queryType": "groupBy"},"store": "azure","format":"csv","key": "druid-reports/","filePath": "druid-reports/","container":"'$bucket'","folderPrefix": ["slug", "reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "TPD Course Consumption Metrics Model","deviceMapping": false}'
+ echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.CourseConsumptionJob","modelParams":{"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]}}},"reportConfig":{"id":"tpd_metrics","labels":{"date":"Date","status":"Batch Status","timespent":"Timespent in mins","courseName":"Course Name","batchName":"Batch Name"},"dateRange":{"staticInterval":"LastDay","granularity":"all"},"metrics":[{"metric":"totalCoursePlays","label":"Total Course Plays (in mins)","druidQuery":{"queryType":"groupBy","dataSource":"summary-events","intervals":"LastDay","aggregations":[{"name":"sum__edata_time_spent","type":"doubleSum","fieldName":"edata_time_spent"}],"dimensions":[{"fieldName":"object_rollup_l1","aliasName":"courseId"},{"fieldName":"uid","aliasName":"userId"},{"fieldName":"context_cdata_id","aliasName":"batchId"}],"filters":[{"type":"equals","dimension":"eid","value":"ME_WORKFLOW_SUMMARY"},{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.app","'$producerEnv'.portal"]},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"equals","dimension":"context_cdata_type","value":"batch"}],"postAggregation":[{"type":"arithmetic","name":"timespent","fields":{"leftField":"sum__edata_time_spent","rightField":60,"rightFieldType":"constant"},"fn":"/"}],"descending":"false"}}],"output":[{"type":"csv","metrics":["timespent"],"dims":[]}],"queryType":"groupBy"},"store":"{{ dp_object_store_type }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Consumption Metrics Model","deviceMapping":false}'
;;
"textbook-progress-report")
- echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.TextBookProgressModel","modelParams":{"reportConfig":{"id":"content_progress_metrics","metrics":[],"labels":{"board":"Board","medium":"Medium","gradeLevel":"Grade","subject":"Subject","resourceType":"Content Type","totalContent": "Total Contents","live":"Live","review":"Review","draft":"Draft","unlisted":"Limited Sharing","application_ecml":"Created on Diksha","video_youtube":"YouTube Content","video_mp4":"Uploaded Videos","application_pdf":"Text Content","application_html":"Uploaded Interactive Content","identifier":"Content ID","creator":"Created By","createdOn":"Creation Date","lastPublishDate":"Last Publish Date","status":"Status","pkgVersion":"Number of times Published","lastPublishedOn":"Pending in current status since","pendingInCurrentStatus":"Pending in current status since"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"content_progress_metrics.csv","postContainer":"'$reportPostContainer'"}},"filter":{"tenantId":"","slugName":""},"store":"azure","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Textbook Progress Metrics Model","deviceMapping":false}'
+ echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.TextBookProgressModel","modelParams":{"reportConfig":{"id":"content_progress_metrics","metrics":[],"labels":{"board":"Board","medium":"Medium","gradeLevel":"Grade","subject":"Subject","resourceType":"Content Type","totalContent":"Total Contents","live":"Live","review":"Review","draft":"Draft","unlisted":"Limited Sharing","application_ecml":"Created on Diksha","video_youtube":"YouTube Content","video_mp4":"Uploaded Videos","application_pdf":"Text Content","application_html":"Uploaded Interactive Content","identifier":"Content ID","creator":"Created By","createdOn":"Creation Date","lastPublishDate":"Last Publish Date","status":"Status","pkgVersion":"Number of times Published","lastPublishedOn":"Pending in current status since","pendingInCurrentStatus":"Pending in current status since"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"content_progress_metrics.csv","postContainer":"'$reportPostContainer'"}},"filter":{"tenantId":"","slugName":""},"store":"{{ dp_object_store_type }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Textbook Progress Metrics Model","deviceMapping":false}'
;;
"audit-metrics-report")
- echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.MetricsAuditJob","modelParams":{"auditConfig":[{"name":"denorm","search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"telemetry-denormalized/raw/","startDate":"'$endDate'","endDate":"'$endDate'"}]},"filters":[{"name":"flags.user_data_retrieved","operator":"EQ","value":true},{"name":"flags.content_data_retrieved","operator":"EQ","value":true},{"name":"flags.device_data_retrieved","operator":"EQ","value":true},{"name":"flags.dialcode_data_retrieved","operator":"EQ","value":true},{"name":"flags.collection_data_retrieved","operator":"EQ","value":true},{"name":"flags.derived_location_retrieved","operator":"EQ","value":true}]},{"name":"failed","search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"failed/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"unique","search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"unique/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"raw","search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"raw/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"channel-raw","search":{"type":"azure","queries":[{"folder":true,"bucket":"'$bucket'","prefix":"channel/*/raw/","startDate":"'$endDate'","endDate":"'$endDate'*.json.gz"}]}},{"name":"channel-summary","search":{"type":"azure","queries":[{"folder":true,"bucket":"'$bucket'","prefix":"channel/*/summary/","startDate":"'$endDate'","endDate":"'$endDate'*.json.gz"}]}},{"name":"derived","search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"derived/wfs/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"telemetry-count","search":{"type":"druid","druidQuery":{"queryType":"timeSeries","dataSource":"telemetry-events","intervals":"LastDay","aggregations":[{"name":"total_count","type":"count","fieldName":"count"}],"descending":"false"}}},{"name":"summary-count","search":{"type":"druid","druidQuery":{"queryType":"timeSeries","dataSource":"summary-events","intervals":"LastDay","aggregations":[{"name":"total_count","type":"count","fieldName":"count"}],"descending":"false"}}}]},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$metricsTopic'"}}],"parallelization":8,"appName":"Metrics Audit"}'
+ echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.MetricsAuditJob","modelParams":{"auditConfig":[{"name":"denorm","search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"telemetry-denormalized/raw/","startDate":"'$endDate'","endDate":"'$endDate'"}]},"filters":[{"name":"flags.user_data_retrieved","operator":"EQ","value":true},{"name":"flags.content_data_retrieved","operator":"EQ","value":true},{"name":"flags.device_data_retrieved","operator":"EQ","value":true},{"name":"flags.dialcode_data_retrieved","operator":"EQ","value":true},{"name":"flags.collection_data_retrieved","operator":"EQ","value":true},{"name":"flags.derived_location_retrieved","operator":"EQ","value":true}]},{"name":"failed","search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"failed/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"unique","search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"unique/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"raw","search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"raw/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"channel-raw","search":{"type":"{{ dp_object_store_type }}","queries":[{"folder":true,"bucket":"'$bucket'","prefix":"channel/*/raw/","startDate":"'$endDate'","endDate":"'$endDate'*.json.gz"}]}},{"name":"channel-summary","search":{"type":"{{ dp_object_store_type }}","queries":[{"folder":true,"bucket":"'$bucket'","prefix":"channel/*/summary/","startDate":"'$endDate'","endDate":"'$endDate'*.json.gz"}]}},{"name":"derived","search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"derived/wfs/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"telemetry-count","search":{"type":"druid","druidQuery":{"queryType":"timeSeries","dataSource":"telemetry-events","intervals":"LastDay","aggregations":[{"name":"total_count","type":"count","fieldName":"count"}],"descending":"false"}}},{"name":"summary-count","search":{"type":"druid","druidQuery":{"queryType":"timeSeries","dataSource":"summary-events","intervals":"LastDay","aggregations":[{"name":"total_count","type":"count","fieldName":"count"}],"descending":"false"}}}]},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$metricsTopic'"}}],"parallelization":8,"appName":"Metrics Audit"}'
;;
"sourcing-metrics")
- echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.sourcing.SourcingMetrics","modelParams": {"reportConfig": {"id": "textbook_report","metrics": [],"labels": {"date": "Date","primaryCategory":"Collection Category","identifier": "Collection ID","name": "Collection Name","medium": "Medium","gradeLevel": "Grade","subject": "Subject","createdOn": "Created On","lastUpdatedOn": "Last Updated On","reportDate": "Report generation date","board": "Board","grade": "Grade","chapters": "Folder Name","totalChapters": "Total number of first level folders","status": "Textbook Status"},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}, {"type": "json","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}]},"druidConfig": {"queryType": "groupBy","dataSource": "content-model-snapshot","intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations": [{"name": "count","type": "count"}],"dimensions": [{"fieldName": "channel","aliasName": "channel"}, {"fieldName": "identifier","aliasName": "identifier","type": "Extraction","outputType": "STRING","extractionFn": [{"type": "javascript","fn": "function(str){return str == null ? null: str.split(\".\")[0]}"}]}, {"fieldName": "name","aliasName": "name"}, {"fieldName": "createdFor","aliasName": "createdFor"}, {"fieldName": "createdOn","aliasName": "createdOn"}, {"fieldName": "lastUpdatedOn","aliasName": "lastUpdatedOn"}, {"fieldName": "board","aliasName": "board"}, {"fieldName": "medium","aliasName": "medium"}, {"fieldName":"primaryCategory","aliasName":"primaryCategory"},{"fieldName": "gradeLevel","aliasName": "gradeLevel"}, {"fieldName": "subject","aliasName": "subject"}, {"fieldName": "status","aliasName": "status"}],"filters": [{"type": "in","dimension": "primaryCategory","values": ["Digital Textbook", "Course", "Content Playlist","Question paper","Question Paper"]}, {"type": "in","dimension": "status","values": ["Live"]}],"postAggregation": [],"descending": "false","limitSpec": {"type": "default","limit": 1000000,"columns": [{"dimension": "count","direction": "descending"}]}},"store": "azure","storageContainer": "'$reportPostContainer'","format": "csv","key": "druid-reports/","filePath": "druid-reports/","container": "'$reportPostContainer'","sparkCassandraConnectionHost": "'$sunbirdPlatformCassandraHost'","folderPrefix": ["slug", "reportName"]},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "Textbook Report Job","deviceMapping": false}'
+ echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.sourcing.SourcingMetrics","modelParams":{"reportConfig":{"id":"textbook_report","metrics":[],"labels":{"date":"Date","primaryCategory":"Collection Category","identifier":"Collection ID","name":"Collection Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","createdOn":"Created On","lastUpdatedOn":"Last Updated On","reportDate":"Report generation date","board":"Board","grade":"Grade","chapters":"Folder Name","totalChapters":"Total number of first level folders","status":"Textbook Status"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]},{"type":"json","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}]},"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"channel","aliasName":"channel"},{"fieldName":"identifier","aliasName":"identifier","type":"Extraction","outputType":"STRING","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.split(\".\")[0]}"}]},{"fieldName":"name","aliasName":"name"},{"fieldName":"createdFor","aliasName":"createdFor"},{"fieldName":"createdOn","aliasName":"createdOn"},{"fieldName":"lastUpdatedOn","aliasName":"lastUpdatedOn"},{"fieldName":"board","aliasName":"board"},{"fieldName":"medium","aliasName":"medium"},{"fieldName":"primaryCategory","aliasName":"primaryCategory"},{"fieldName":"gradeLevel","aliasName":"gradeLevel"},{"fieldName":"subject","aliasName":"subject"},{"fieldName":"status","aliasName":"status"}],"filters":[{"type":"in","dimension":"primaryCategory","values":["Digital Textbook","Course","Content Playlist","Question paper","Question Paper"]},{"type":"in","dimension":"status","values":["Live"]}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"store":"{{ dp_object_store_type }}","storageContainer":"'$reportPostContainer'","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$reportPostContainer'","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Textbook Report Job","deviceMapping":false}'
;;
"druid-dataset")
- echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.OnDemandDruidExhaustJob","modelParams":{"store":"azure","container":"'$reportPostContainer'","key":"ml_reports/","format":"csv"},"output":[{"to": "console","params": {"printEvent": false}}],"parallelization":8,"appName":"ML Druid Data Model"}'
+ echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.OnDemandDruidExhaustJob","modelParams":{"store":"{{ dp_object_store_type }}","container":"'$reportPostContainer'","key":"ml_reports/","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"ML Druid Data Model"}'
;;
"*")
echo "Unknown model code"
diff --git a/ansible/roles/data-products-deploy/templates/model-config.json.j2 b/ansible/roles/data-products-deploy/templates/model-config.json.j2
index 4594a1978a..bb170b2193 100644
--- a/ansible/roles/data-products-deploy/templates/model-config.json.j2
+++ b/ansible/roles/data-products-deploy/templates/model-config.json.j2
@@ -1,7 +1,7 @@
{
"wfs": {
"search": {
- "type": "azure",
+ "type": "{{dp_object_store_type}}",
"queries": [
{
"bucket": "{{ bucket }}",
@@ -20,11 +20,16 @@
"model": "org.ekstep.analytics.model.WorkflowSummary",
"modelParams": {
"apiVersion": "v2",
- "parallelization": 32
+ "parallelization": 32,
+ "storageKeyConfig":"storage.key.config",
+ "storageSecretConfig":"storage.secret.config",
+ "storageContainer":"{{reports_container}}",
+ "storageEndpoint":"{{dp_storage_endpoint_config}}",
+ "store":"{{ dp_object_store_type }}"
},
"output": [
{
- "to": "azure",
+ "to": "{{dp_object_store_type}}",
"params": {
"bucket": "{{ bucket }}",
"key": "{{ job_manager_tmp_dir }}/wfs/$(date --date yesterday '+%Y-%m-%d')"
@@ -44,7 +49,7 @@
},
"video-streaming": {
"search": {
- "type": "azure"
+ "type": "{{dp_object_store_type}}"
},
"model": "org.ekstep.analytics.job.VideoStreamingJob",
"modelParams": {
@@ -68,6 +73,11 @@
},
"model": "org.ekstep.analytics.job.report.StateAdminReportJob",
"modelParams": {
+ "storageKeyConfig":"storage.key.config",
+ "storageSecretConfig":"storage.secret.config",
+ "storageContainer":"{{reports_container}}",
+ "storageEndpoint":"{{dp_storage_endpoint_config}}",
+ "store":"{{ dp_object_store_type }}",
"sparkCassandraConnectionHost": "{{core_cassandra_host}}",
"sparkElasticsearchConnectionHost": "{{sunbird_es_host}}"
},
@@ -89,6 +99,11 @@
},
"model": "org.ekstep.analytics.job.report.StateAdminGeoReportJob",
"modelParams": {
+ "storageKeyConfig":"storage.key.config",
+ "storageSecretConfig":"storage.secret.config",
+ "storageContainer":"{{reports_container}}",
+ "storageEndpoint":"{{dp_storage_endpoint_config}}",
+ "store":"{{ dp_object_store_type }}",
"sparkCassandraConnectionHost": "{{core_cassandra_host}}",
"sparkElasticsearchConnectionHost": "{{sunbird_es_host}}"
},
@@ -110,6 +125,11 @@
},
"model": "org.ekstep.analytics.updater.UpdateContentRating",
"modelParams": {
+ "store": "{{dp_object_store_type}}",
+ "storageKeyConfig":"storage.key.config",
+ "storageSecretConfig":"storage.secret.config",
+ "storageContainer":"{{reports_container}}",
+ "storageEndpoint":"{{dp_storage_endpoint_config}}",
"startDate": "$(date --date yesterday '+%Y-%m-%d')",
"endDate": "$(date '+%Y-%m-%d')"
},
@@ -139,6 +159,11 @@
"pushMetrics": true,
"brokerList": "{{ brokerlist }}",
"topic": "{{ analytics_metrics_topic }}",
+ "storageKeyConfig":"storage.key.config",
+ "storageSecretConfig":"storage.secret.config",
+ "storageContainer":"{{reports_container}}",
+ "storageEndpoint":"{{dp_storage_endpoint_config}}",
+ "store":"{{ dp_object_store_type }}",
"model": [
{
"model": "WorkFlowSummaryModel",
@@ -201,6 +226,11 @@
},
"model": "org.ekstep.analytics.model.ExperimentDefinitionModel",
"modelParams": {
+ "store": "{{dp_object_store_type}}",
+ "storageKeyConfig":"storage.key.config",
+ "storageSecretConfig":"storage.secret.config",
+ "storageContainer":"{{reports_container}}",
+ "storageEndpoint":"{{dp_storage_endpoint_config}}",
"sparkElasticsearchConnectionHost": "{{ lp_composite_search_host }}"
},
"output": [
@@ -297,7 +327,11 @@
"tenantId": "",
"slugName": ""
},
- "store": "azure",
+ "store": "{{dp_object_store_type}}",
+ "storageKeyConfig":"storage.key.config",
+ "storageSecretConfig":"storage.secret.config",
+ "storageContainer":"{{reports_container}}",
+ "storageEndpoint":"{{dp_storage_endpoint_config}}",
"format": "csv",
"key": "druid-reports/",
"filePath": "druid-reports/",
@@ -346,7 +380,11 @@
"limit": 10000
}
},
- "store": "azure",
+ "store": "{{dp_object_store_type}}",
+ "storageKeyConfig":"storage.key.config",
+ "storageSecretConfig":"storage.secret.config",
+ "storageContainer":"{{reports_container}}",
+ "storageEndpoint":"{{dp_storage_endpoint_config}}",
"format":"csv",
"key": "druid-reports/",
"filePath": "druid-reports/",
@@ -457,7 +495,11 @@
}],
"queryType": "groupBy"
},
- "store": "azure",
+ "store": "{{dp_object_store_type}}",
+ "storageKeyConfig":"storage.key.config",
+ "storageSecretConfig":"storage.secret.config",
+ "storageContainer":"{{reports_container}}",
+ "storageEndpoint":"{{dp_storage_endpoint_config}}",
"format":"csv",
"key": "druid-reports/",
"filePath": "druid-reports/",
@@ -482,11 +524,16 @@
},
"model": "org.ekstep.analytics.model.MetricsAuditJob",
"modelParams": {
+ "store": "{{dp_object_store_type}}",
+ "storageKeyConfig":"storage.key.config",
+ "storageSecretConfig":"storage.secret.config",
+ "storageContainer":"{{reports_container}}",
+ "storageEndpoint":"{{dp_storage_endpoint_config}}",
"auditConfig": [
{
"name": "denorm",
"search": {
- "type": "azure",
+ "type": "{{dp_object_store_type}}",
"queries": [
{
"bucket": "{{ bucket }}",
@@ -532,7 +579,7 @@
{
"name": "failed",
"search": {
- "type": "azure",
+ "type": "{{dp_object_store_type}}",
"queries": [
{
"bucket": "{{ bucket }}",
@@ -546,7 +593,7 @@
{
"name": "unique",
"search": {
- "type": "azure",
+ "type": "{{dp_object_store_type}}",
"queries": [
{
"bucket": "{{ bucket }}",
@@ -560,7 +607,7 @@
{
"name": "raw",
"search": {
- "type": "azure",
+ "type": "{{dp_object_store_type}}",
"queries": [
{
"bucket": "{{ bucket }}",
@@ -574,7 +621,7 @@
{
"name": "channel-raw",
"search": {
- "type": "azure",
+ "type": "{{dp_object_store_type}}",
"queries": [
{
"folder": true,
@@ -589,7 +636,7 @@
{
"name": "channel-summary",
"search": {
- "type": "azure",
+ "type": "{{dp_object_store_type}}",
"queries": [
{
"folder": true,
@@ -604,7 +651,7 @@
{
"name": "derived",
"search": {
- "type": "azure",
+ "type": "{{dp_object_store_type}}",
"queries": [
{
"bucket": "{{ bucket }}",
diff --git a/ansible/roles/data-products-deploy/templates/model-dock-config.j2 b/ansible/roles/data-products-deploy/templates/model-dock-config.j2
index 20d82dbfb5..f720f4687e 100644
--- a/ansible/roles/data-products-deploy/templates/model-dock-config.j2
+++ b/ansible/roles/data-products-deploy/templates/model-dock-config.j2
@@ -15,16 +15,16 @@ config() {
if [ ! -z "$3" ]; then inputBucket=$3; fi
case "$1" in
"content-details")
- echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.sourcing.ContentDetailsReport","modelParams":{"tenantId":"","slug":"","reportConfig":{"id":"content_report","metrics":[],"labels":{"programName":"Project Name","programId":"Project ID","contentId":"Content/Question ID","contentName":"Content/Question Name","mimeType":"MimeType","chapterId":"Folder ID","contentStatus":"Content/Question Status","creator":"Creator Name","createdBy":"CreatedBy ID","date":"Date","identifier":"Collection/Question Set ID","name":"Collection/Question Set Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","board":"Board","grade":"Grade","chapters":"Chapter Name","status":"Textbook Status","objectType":"Object Type","primaryCategory":"Primary category","topic":"Topic","learningOutcome":"Learning Outcome","addedFromLibrary":"Added from library","contentType":"Content Type"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]},{"type":"json","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}]},"contentQuery":{"queryType":"groupBy","dataSource":"vdn-content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"},{"fieldName":"name","aliasName":"name"},{"fieldName":"unitIdentifiers","aliasName":"unitIdentifiers"},{"fieldName":"collectionId","aliasName":"collectionId"},{"fieldName":"createdBy","aliasName":"createdBy"},{"fieldName":"creator","aliasName":"creator"},{"fieldName":"mimeType","aliasName":"mimeType"},{"fieldName":"topic","aliasName":"topic"},{"fieldName":"learningOutcome","aliasName":"learningOutcome"},{"fieldName":"primaryCategory","aliasName":"contentType"}],"filters":[{"type":"notequals","dimension":"contentType","value":"TextBook"},{"type":"in","dimension":"status","values":["Live"]},{"type":"isnotnull","dimension":"collectionId"}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"textbookQuery":{"queryType":"groupBy","dataSource":"vdn-content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"programId","aliasName":"programId"},{"fieldName":"identifier","aliasName":"identifier"},{"fieldName":"name","aliasName":"name"},{"fieldName":"board","aliasName":"board"},{"fieldName":"medium","aliasName":"medium"},{"fieldName":"gradeLevel","aliasName":"gradeLevel"},{"fieldName":"subject","aliasName":"subject"},{"fieldName":"status","aliasName":"status"},{"fieldName":"acceptedContents","aliasName":"acceptedContents"},{"fieldName":"acceptedContributions","aliasName":"acceptedContributions"},{"fieldName":"rejectedContents","aliasName":"rejectedContents"},{"fieldName":"rejectedContributions","aliasName":"rejectedContributions"},{"fieldName":"primaryCategory","aliasName":"primaryCategory"},{"fieldName":"objectType","aliasName":"objectType"},{"fieldName":"reusedContributions","aliasName":"reusedContributions"}],"filters":[{"type":"in","dimension":"primaryCategory","values":["Digital Textbook","Course","Content Playlist","Question paper","Question Paper","Exam Question Set","Practice Set","Demo Practice Question Set"]},{"type":"isnotnull","dimension":"programId"},{"type":"in","dimension":"status","values":["Draft"]},{"type":"equals","dimension":"channel","value":"channelId"}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"store":"azure","storageKeyConfig":"azure_storage_key","storageSecretConfig":"azure_storage_secret","storageContainer":"'$reportPostContainer'","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$reportPostContainer'","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Content Report Job","deviceMapping":false}'
+ echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.sourcing.ContentDetailsReport","modelParams":{"tenantId":"","slug":"","reportConfig":{"id":"content_report","metrics":[],"labels":{"programName":"Project Name","programId":"Project ID","contentId":"Content/Question ID","contentName":"Content/Question Name","mimeType":"MimeType","chapterId":"Folder ID","contentStatus":"Content/Question Status","creator":"Creator Name","createdBy":"CreatedBy ID","date":"Date","identifier":"Collection/Question Set ID","name":"Collection/Question Set Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","board":"Board","grade":"Grade","chapters":"Chapter Name","status":"Textbook Status","objectType":"Object Type","primaryCategory":"Primary category","topic":"Topic","learningOutcome":"Learning Outcome","addedFromLibrary":"Added from library","contentType":"Content Type"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]},{"type":"json","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}]},"contentQuery":{"queryType":"groupBy","dataSource":"vdn-content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"},{"fieldName":"name","aliasName":"name"},{"fieldName":"unitIdentifiers","aliasName":"unitIdentifiers"},{"fieldName":"collectionId","aliasName":"collectionId"},{"fieldName":"createdBy","aliasName":"createdBy"},{"fieldName":"creator","aliasName":"creator"},{"fieldName":"mimeType","aliasName":"mimeType"},{"fieldName":"topic","aliasName":"topic"},{"fieldName":"learningOutcome","aliasName":"learningOutcome"},{"fieldName":"primaryCategory","aliasName":"contentType"}],"filters":[{"type":"notequals","dimension":"contentType","value":"TextBook"},{"type":"in","dimension":"status","values":["Live"]},{"type":"isnotnull","dimension":"collectionId"}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"textbookQuery":{"queryType":"groupBy","dataSource":"vdn-content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"programId","aliasName":"programId"},{"fieldName":"identifier","aliasName":"identifier"},{"fieldName":"name","aliasName":"name"},{"fieldName":"board","aliasName":"board"},{"fieldName":"medium","aliasName":"medium"},{"fieldName":"gradeLevel","aliasName":"gradeLevel"},{"fieldName":"subject","aliasName":"subject"},{"fieldName":"status","aliasName":"status"},{"fieldName":"acceptedContents","aliasName":"acceptedContents"},{"fieldName":"acceptedContributions","aliasName":"acceptedContributions"},{"fieldName":"rejectedContents","aliasName":"rejectedContents"},{"fieldName":"rejectedContributions","aliasName":"rejectedContributions"},{"fieldName":"primaryCategory","aliasName":"primaryCategory"},{"fieldName":"objectType","aliasName":"objectType"},{"fieldName":"reusedContributions","aliasName":"reusedContributions"}],"filters":[{"type":"in","dimension":"primaryCategory","values":["Digital Textbook","Course","Content Playlist","Question paper","Question Paper","Exam Question Set","Practice Set","Demo Practice Question Set"]},{"type":"isnotnull","dimension":"programId"},{"type":"in","dimension":"status","values":["Draft"]},{"type":"equals","dimension":"channel","value":"channelId"}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"store":"{{dp_object_store_type}}","storageKeyConfig":"azure_storage_key","storageSecretConfig":"azure_storage_secret","storageContainer":"'$reportPostContainer'","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$reportPostContainer'","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Content Report Job","deviceMapping":false}'
;;
"sourcing-summary-report")
- echo '{"search": {"type": "none"}, "model": "org.ekstep.analytics.job.report.SourcingSummaryReport", "modelParams": {"storageKeyConfig":"druid_storage_account_key", "storageSecretConfig":"druid_storage_account_secret", "dataSource": "sourcing-summary-snapshot", "druidHost": "'$druidRollupHost'", "druidSegmentUrl":"'$druidRollupHost'/druid/coordinator/v1/metadata/datasources/sourcing-model-snapshot/segments", "deleteSegmentUrl": "'$druidRollupHost'/druid/coordinator/v1/datasources/sourcing-model-snapshot/segments/", "druidIngestionUrl": "'$druidRollupHost'/druid/indexer/v1/task", "specPath": "/mount/data/analytics/scripts/sourcing-ingestion-spec.json", "dbName": "opensaberdb", "tables": {"programTable": "program", "nominationTable": "nomination"}, "druidQuery": {"queryType": "groupBy", "dataSource": "vdn-content-model-snapshot", "intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00", "aggregations": [{"name": "count", "type": "count"}], "dimensions": [{"fieldName": "primaryCategory", "aliasName": "primaryCategory"}, {"fieldName": "createdBy", "aliasName": "createdBy"}], "filters": [{"type": "equals", "dimension": "objectType", "value": "Content"}, {"type": "equals", "dimension": "sampleContent", "value": "false"}], "postAggregation": [], "descending": "false", "limitSpec": {"type": "default", "limit": 1000000, "columns": [{"dimension": "count", "direction": "descending"}]}}, "reportConfig": {"id": "sourcing", "metrics": [], "labels": {}, "output": [{"type": "json", "dims": ["identifier", "channel", "name"], "fileParameters": ["id", "dims"]}]}, "store": "azure", "format": "json", "folderPrefix": ["slug", "reportName"]}, "output": [{"to": "console", "params": {"printEvent": false}}], "parallelization": 8, "appName": "Sourcing Summary Report Job", "deviceMapping": false}'
+ echo '{"search": {"type": "none"}, "model": "org.ekstep.analytics.job.report.SourcingSummaryReport", "modelParams": {"storageKeyConfig":"druid_storage_account_key", "storageSecretConfig":"druid_storage_account_secret", "dataSource": "sourcing-summary-snapshot", "druidHost": "'$druidRollupHost'", "druidSegmentUrl":"'$druidRollupHost'/druid/coordinator/v1/metadata/datasources/sourcing-model-snapshot/segments", "deleteSegmentUrl": "'$druidRollupHost'/druid/coordinator/v1/datasources/sourcing-model-snapshot/segments/", "druidIngestionUrl": "'$druidRollupHost'/druid/indexer/v1/task", "specPath": "/mount/data/analytics/scripts/sourcing-ingestion-spec.json", "dbName": "opensaberdb", "tables": {"programTable": "program", "nominationTable": "nomination"}, "druidQuery": {"queryType": "groupBy", "dataSource": "vdn-content-model-snapshot", "intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00", "aggregations": [{"name": "count", "type": "count"}], "dimensions": [{"fieldName": "primaryCategory", "aliasName": "primaryCategory"}, {"fieldName": "createdBy", "aliasName": "createdBy"}], "filters": [{"type": "equals", "dimension": "objectType", "value": "Content"}, {"type": "equals", "dimension": "sampleContent", "value": "false"}], "postAggregation": [], "descending": "false", "limitSpec": {"type": "default", "limit": 1000000, "columns": [{"dimension": "count", "direction": "descending"}]}}, "reportConfig": {"id": "sourcing", "metrics": [], "labels": {}, "output": [{"type": "json", "dims": ["identifier", "channel", "name"], "fileParameters": ["id", "dims"]}]}, "store": "{{dp_object_store_type}}", "format": "json", "folderPrefix": ["slug", "reportName"]}, "output": [{"to": "console", "params": {"printEvent": false}}], "parallelization": 8, "appName": "Sourcing Summary Report Job", "deviceMapping": false}'
;;
"funnel-report")
- echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.job.report.FunnelReport","modelParams": {"contributionConfig": {"contentRequest": {"request": {"filters": {"programId": "programIdentifier","objectType": "content","status": ["Draft", "Live", "Review"],"mimeType": "application/vnd.ekstep.content-collection"},"fields": ["acceptedContents", "rejectedContents"],"limit": 10000}},"correctionsPendingRequest": {"request": {"filters": {"objectType": ["content","questionset"],"status": "Draft","prevStatus": "Live","programId": "programIdentifier","mimeType": {"!=": "application/vnd.ekstep.content-collection"},"contentType": {"!=": "Asset"}},"not_exists": ["sampleContent"],"facets": ["createdBy"],"limit": 0}},"contributionRequest": {"request": {"filters": {"objectType": ["content","questionset"],"status": ["Live"],"programId": "programIdentifier","mimeType": {"!=": "application/vnd.ekstep.content-collection"},"contentType": {"!=": "Asset"}},"not_exists": ["sampleContent"],"facets": ["createdBy"],"limit": 0}}},"reportConfig": {"id": "funnel_report","metrics": [],"labels": {"reportDate": "Report generation date","visitors": "No. of users opening the project","projectName": "Project Name","initiatedNominations": "No. of initiated nominations","rejectedNominations": "No. of rejected nominations","pendingNominations": "No. of nominations pending review","acceptedNominations": "No. of accepted nominations to the project","noOfContributors": "No. of contributors to the project","noOfContributions": "No. of contributions to the project","pendingContributions": "No. of contributions pending review","approvedContributions": "No. of approved contributions"},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}, {"type": "json","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}]},"store": "azure","format": "csv","key": "druid-reports/","filePath": "druid-reports/","container": "'$reportPostContainer'","folderPrefix": ["slug", "reportName"]},"sparkCassandraConnectionHost": "'$sunbirdPlatformCassandraHost'","druidConfig": {"queryType": "timeseries","dataSource": "telemetry-events-syncts","intervals": "startdate/enddate","aggregations": [{"name": "visitors","type": "count","fieldName": "actor_id"}],"filters": [{"type": "equals","dimension": "context_cdata_id","value": "program_id"}, {"type": "equals","dimension": "edata_pageid","value": "contribution_project_contributions"}, {"type": "equals","dimension": "context_pdata_pid","value": "creation-portal.programs"}, {"type": "equals","dimension": "context_cdata_type","value": "project"}, {"type": "equals","dimension": "context_env","value": "creation-portal"}, {"type": "equals","dimension": "eid","value": "IMPRESSION"}],"postAggregation": [],"descending": "false","limitSpec": {"type": "default","limit": 1000000,"columns": [{"dimension": "count","direction": "descending"}]}},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "Funnel Report Job","deviceMapping": false}'
+ echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.job.report.FunnelReport","modelParams": {"contributionConfig": {"contentRequest": {"request": {"filters": {"programId": "programIdentifier","objectType": "content","status": ["Draft", "Live", "Review"],"mimeType": "application/vnd.ekstep.content-collection"},"fields": ["acceptedContents", "rejectedContents"],"limit": 10000}},"correctionsPendingRequest": {"request": {"filters": {"objectType": ["content","questionset"],"status": "Draft","prevStatus": "Live","programId": "programIdentifier","mimeType": {"!=": "application/vnd.ekstep.content-collection"},"contentType": {"!=": "Asset"}},"not_exists": ["sampleContent"],"facets": ["createdBy"],"limit": 0}},"contributionRequest": {"request": {"filters": {"objectType": ["content","questionset"],"status": ["Live"],"programId": "programIdentifier","mimeType": {"!=": "application/vnd.ekstep.content-collection"},"contentType": {"!=": "Asset"}},"not_exists": ["sampleContent"],"facets": ["createdBy"],"limit": 0}}},"reportConfig": {"id": "funnel_report","metrics": [],"labels": {"reportDate": "Report generation date","visitors": "No. of users opening the project","projectName": "Project Name","initiatedNominations": "No. of initiated nominations","rejectedNominations": "No. of rejected nominations","pendingNominations": "No. of nominations pending review","acceptedNominations": "No. of accepted nominations to the project","noOfContributors": "No. of contributors to the project","noOfContributions": "No. of contributions to the project","pendingContributions": "No. of contributions pending review","approvedContributions": "No. of approved contributions"},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}, {"type": "json","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}]},"store": "{{dp_object_store_type}}","format": "csv","key": "druid-reports/","filePath": "druid-reports/","container": "'$reportPostContainer'","folderPrefix": ["slug", "reportName"]},"sparkCassandraConnectionHost": "'$sunbirdPlatformCassandraHost'","druidConfig": {"queryType": "timeseries","dataSource": "telemetry-events-syncts","intervals": "startdate/enddate","aggregations": [{"name": "visitors","type": "count","fieldName": "actor_id"}],"filters": [{"type": "equals","dimension": "context_cdata_id","value": "program_id"}, {"type": "equals","dimension": "edata_pageid","value": "contribution_project_contributions"}, {"type": "equals","dimension": "context_pdata_pid","value": "creation-portal.programs"}, {"type": "equals","dimension": "context_cdata_type","value": "project"}, {"type": "equals","dimension": "context_env","value": "creation-portal"}, {"type": "equals","dimension": "eid","value": "IMPRESSION"}],"postAggregation": [],"descending": "false","limitSpec": {"type": "default","limit": 1000000,"columns": [{"dimension": "count","direction": "descending"}]}},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "Funnel Report Job","deviceMapping": false}'
;;
"sourcing-metrics")
- echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.sourcing.SourcingMetrics","modelParams": {"reportConfig": {"id": "textbook_report","metrics": [],"labels": {"date": "Date","identifier": "Textbook ID","name": "Textbook Name","medium": "Medium","gradeLevel": "Grade","subject": "Subject","createdOn": "Created On","lastUpdatedOn": "Last Updated On","reportDate": "Report generation date","board": "Board","grade": "Grade","chapters": "Chapter Name","totalChapters": "Total number of chapters (first level sections of ToC)","status": "Textbook Status"},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}, {"type": "json","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}]},"druidConfig": {"queryType": "groupBy","dataSource": "content-model-snapshot","intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations": [{"name": "count","type": "count"}],"dimensions": [{"fieldName": "channel","aliasName": "channel"}, {"fieldName": "identifier","aliasName": "identifier","type": "Extraction","outputType": "STRING","extractionFn": [{"type": "javascript","fn": "function(str){return str == null ? null: str.split(\".\")[0]}"}]}, {"fieldName": "name","aliasName": "name"}, {"fieldName": "createdFor","aliasName": "createdFor"}, {"fieldName": "createdOn","aliasName": "createdOn"}, {"fieldName": "lastUpdatedOn","aliasName": "lastUpdatedOn"}, {"fieldName": "board","aliasName": "board"}, {"fieldName": "medium","aliasName": "medium"}, {"fieldName": "gradeLevel","aliasName": "gradeLevel"}, {"fieldName": "subject","aliasName": "subject"}, {"fieldName": "status","aliasName": "status"}],"filters": [{"type": "equals","dimension": "contentType","value": "TextBook"}, {"type": "in","dimension": "status","values": ["Live"]}],"postAggregation": [],"descending": "false","limitSpec": {"type": "default","limit": 1000000,"columns": [{"dimension": "count","direction": "descending"}]}},"store": "azure","storageContainer": "'$reportPostContainer'","format": "csv","key": "druid-reports/","filePath": "druid-reports/","container": "'$reportPostContainer'","sparkCassandraConnectionHost": "'$sunbirdPlatformCassandraHost'","folderPrefix": ["slug", "reportName"]},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "Textbook Report Job","deviceMapping": false}'
+ echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.sourcing.SourcingMetrics","modelParams": {"reportConfig": {"id": "textbook_report","metrics": [],"labels": {"date": "Date","identifier": "Textbook ID","name": "Textbook Name","medium": "Medium","gradeLevel": "Grade","subject": "Subject","createdOn": "Created On","lastUpdatedOn": "Last Updated On","reportDate": "Report generation date","board": "Board","grade": "Grade","chapters": "Chapter Name","totalChapters": "Total number of chapters (first level sections of ToC)","status": "Textbook Status"},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}, {"type": "json","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}]},"druidConfig": {"queryType": "groupBy","dataSource": "content-model-snapshot","intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations": [{"name": "count","type": "count"}],"dimensions": [{"fieldName": "channel","aliasName": "channel"}, {"fieldName": "identifier","aliasName": "identifier","type": "Extraction","outputType": "STRING","extractionFn": [{"type": "javascript","fn": "function(str){return str == null ? null: str.split(\".\")[0]}"}]}, {"fieldName": "name","aliasName": "name"}, {"fieldName": "createdFor","aliasName": "createdFor"}, {"fieldName": "createdOn","aliasName": "createdOn"}, {"fieldName": "lastUpdatedOn","aliasName": "lastUpdatedOn"}, {"fieldName": "board","aliasName": "board"}, {"fieldName": "medium","aliasName": "medium"}, {"fieldName": "gradeLevel","aliasName": "gradeLevel"}, {"fieldName": "subject","aliasName": "subject"}, {"fieldName": "status","aliasName": "status"}],"filters": [{"type": "equals","dimension": "contentType","value": "TextBook"}, {"type": "in","dimension": "status","values": ["Live"]}],"postAggregation": [],"descending": "false","limitSpec": {"type": "default","limit": 1000000,"columns": [{"dimension": "count","direction": "descending"}]}},"store": "{{dp_object_store_type}}","storageContainer": "'$reportPostContainer'","format": "csv","key": "druid-reports/","filePath": "druid-reports/","container": "'$reportPostContainer'","sparkCassandraConnectionHost": "'$sunbirdPlatformCassandraHost'","folderPrefix": ["slug", "reportName"]},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "Textbook Report Job","deviceMapping": false}'
;;
"*")
echo "Unknown model code"
diff --git a/ansible/roles/data-products-deploy/templates/sourcing-ingestion-spec.j2 b/ansible/roles/data-products-deploy/templates/sourcing-ingestion-spec.j2
new file mode 100644
index 0000000000..41bb51afba
--- /dev/null
+++ b/ansible/roles/data-products-deploy/templates/sourcing-ingestion-spec.j2
@@ -0,0 +1,151 @@
+{
+ "type": "index",
+ "spec": {
+ "dataSchema": {
+ "dataSource": "sourcing-summary-snapshot",
+ "parser": {
+ "type": "string",
+ "parseSpec": {
+ "format": "json",
+ "flattenSpec": {
+ "useFieldDiscovery": false,
+ "fields": [
+ {
+ "type": "root",
+ "name": "program_id",
+ "expr": "program_id"
+ },
+ {
+ "type": "root",
+ "name": "status",
+ "expr": "status"
+ },
+ {
+ "type": "root",
+ "name": "rootorg_id",
+ "expr": "rootorg_id"
+ },
+ {
+ "type": "root",
+ "name": "user_id",
+ "expr": "user_id"
+ },
+ {
+ "type": "root",
+ "name": "osid",
+ "expr": "osid"
+ },
+ {
+ "type": "root",
+ "name": "user_type",
+ "expr": "user_type"
+ },
+ {
+ "type": "root",
+ "name": "contributor_id",
+ "expr": "contributor_id"
+ },
+ {
+ "type": "root",
+ "name": "total_contributed_content",
+ "expr": "total_contributed_content"
+ },
+ {
+ "type": "root",
+ "name": "primary_category",
+ "expr": "primary_category"
+ },
+ {
+ "type": "root",
+ "name": "created_by",
+ "expr": "created_by"
+ }
+ ]
+ },
+ "dimensionsSpec": {
+ "dimensions": [
+ {
+ "type": "string",
+ "name": "program_id"
+ },
+ {
+ "type": "string",
+ "name": "status"
+ },
+ {
+ "type": "string",
+ "name": "rootorg_id"
+ },
+ {
+ "type": "string",
+ "name": "user_id"
+ },
+ {
+ "type": "string",
+ "name": "osid"
+ },
+ {
+ "type": "string",
+ "name": "user_type"
+ },
+ {
+ "type": "string",
+ "name": "contributor_id"
+ },
+ {
+ "type": "string",
+ "name": "primary_category"
+ },
+ {
+ "type": "string",
+ "name": "created_by"
+ }
+ ],
+ "dimensionsExclusions": []
+ },
+ "timestampSpec": {
+ "column": "timestamp",
+ "format": "auto"
+ }
+ }
+ },
+ "metricsSpec": [
+ {
+ "name": "total_count",
+ "type": "count"
+ }
+ ],
+ "granularitySpec": {
+ "type": "uniform",
+ "segmentGranularity": "day",
+ "queryGranularity": "none",
+ "rollup": true
+ }
+ },
+ "ioConfig": {
+ "type": "index",
+ "firehose": {
+{% if dp_object_store_type == "azure" %}
+ "type": "static-azure-blobstore",
+ "blobs": [
+ {
+ "container": "{{reports_container}}",
+ "path": "/sourcing/SourcingSummaryReport.json"
+ }
+ ],
+{% elif (dp_object_store_type == "oci") %}
+ "type": "static-s3",
+ "uris": [ "s3://{{reports_container}}/sourcing/SourcingSummaryReport.json"],
+{% endif %}
+ "fetchTimeout": 300000
+ }
+ },
+ "tuningConfig": {
+ "type": "index",
+ "targetPartitionSize": 5000000,
+ "maxRowsInMemory": 25000,
+ "forceExtendableShardSpecs": false,
+ "logParseExceptions": true
+ }
+ }
+}
diff --git a/ansible/roles/data-products-deploy/templates/start-jobmanager.j2 b/ansible/roles/data-products-deploy/templates/start-jobmanager.j2
index 2e613b9866..a3a156cfda 100644
--- a/ansible/roles/data-products-deploy/templates/start-jobmanager.j2
+++ b/ansible/roles/data-products-deploy/templates/start-jobmanager.j2
@@ -5,12 +5,12 @@ export DP_LOGS={{ analytics.home }}/logs/data-products
export SERVICE_LOGS={{ analytics.home }}/logs/services
export JM_HOME={{ analytics.home }}/job-manager
-export azure_storage_key={{sunbird_private_storage_account_name}}
-export azure_storage_secret={{sunbird_private_storage_account_key}}
-export reports_azure_storage_key={{sunbird_private_storage_account_name}}
-export reports_azure_storage_secret={{sunbird_private_storage_account_key}}
-export druid_storage_account_key={{sunbird_public_storage_account_name}}
-export druid_storage_account_secret={{sunbird_public_storage_account_key}}
+export azure_storage_key={{cloud_private_storage_accountname}}
+export azure_storage_secret={{cloud_private_storage_secret}}
+export reports_azure_storage_key={{cloud_private_storage_accountname}}
+export reports_azure_storage_secret={{cloud_private_storage_secret}}
+export druid_storage_account_key={{cloud_public_storage_accountname}}
+export druid_storage_account_secret={{cloud_public_storage_secret}}
export heap_conf_str={{ spark.heap_conf_str }}
today=$(date "+%Y-%m-%d")
diff --git a/ansible/roles/data-products-deploy/templates/submit-script.j2 b/ansible/roles/data-products-deploy/templates/submit-script.j2
index e8341dc1e8..0e629a4ce8 100644
--- a/ansible/roles/data-products-deploy/templates/submit-script.j2
+++ b/ansible/roles/data-products-deploy/templates/submit-script.j2
@@ -1,6 +1,7 @@
#!/usr/bin/env bash
## Job to run daily
+
cd "{{ analytics_cluster.home }}"
source model-config.sh
today=$(date "+%Y-%m-%d")
@@ -79,7 +80,15 @@ submit_cluster_job() {
requestBody=${clusterConfig/'"className": "org.ekstep.analytics.job.JobExecutor"'/$argsStr}
finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable}
echo $finalRequestBody
- response=$(curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: admin_name }}")
+{% if dp_object_store_type == "azure" %}
+{
+ curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: {{ admin_name }}"
+}
+{% elif (dp_object_store_type == "oci") %}
+{
+ curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_bds_url }}:8998/batches' -H "X-Requested-By: {{ admin_name }}"
+}
+{% endif %}
echo "Submitted job for batchNumer $i below is the response"
echo $response
}
@@ -118,7 +127,15 @@ if [ "$mode" = "via-partition" ]; then
requestBody=${clusterConfig/'"className": "org.ekstep.analytics.job.JobExecutor"'/$argsStr}
finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable}
echo $finalRequestBody
+{% if dp_object_store_type == "azure" %}
+{
curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: {{ admin_name }}"
+}
+{% elif (dp_object_store_type == "oci") %}
+{
+ curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_bds_url }}:8998/batches' -H "X-Requested-By: {{ admin_name }}"
+}
+{% endif %}
done
elif [ "$mode" = "parallel-jobs" ]; then
@@ -157,8 +174,15 @@ elif [ "$mode" = "selected-partition" ]; then
requestBody=${clusterConfig/'"className": "org.ekstep.analytics.job.JobExecutor"'/$argsStr}
finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable}
echo $finalRequestBody
+{% if dp_object_store_type == "azure" %}
+{
curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: {{ admin_name }}"
-
+}
+{% elif (dp_object_store_type == "oci") %}
+{
+ curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_bds_url }}:8998/batches' -H "X-Requested-By: {{ admin_name }}"
+}
+{% endif %}
else
if [ -z "$start_date" ]; then
echo "Running $job without partition via run-job."
@@ -179,5 +203,14 @@ else
requestBody=${clusterConfig/'"className": "org.ekstep.analytics.job.JobExecutor"'/$argsStr}
finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable}
echo $finalRequestBody
- curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: {{ admin_name }}"
+{% if dp_object_store_type == "azure" %}
+{
+ curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: {{ admin_name }}"
+}
+{% elif (dp_object_store_type == "oci") %}
+{
+ curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_bds_url }}:8998/batches' -H "X-Requested-By: {{ admin_name }}"
+}
+{% endif %}
+
fi
diff --git a/ansible/roles/es-azure-snapshot/defaults/main.yml b/ansible/roles/es-azure-snapshot/defaults/main.yml
index 39ca274c0e..b493109086 100644
--- a/ansible/roles/es-azure-snapshot/defaults/main.yml
+++ b/ansible/roles/es-azure-snapshot/defaults/main.yml
@@ -1,7 +1,7 @@
snapshot_create_request_body: {
type: azure,
settings: {
- container: "{{ es_backup_storage }}",
+ container: "{{ cloud_storage_esbackup_foldername }}",
base_path: "{{ snapshot_base_path }}_{{ base_path_date }}"
}
}
@@ -10,12 +10,12 @@ snapshot_create_request_body: {
es_snapshot_host: "localhost"
snapshot_base_path: "default"
-es_azure_backup_container_name: "elasticsearch-snapshots"
+cloud_storage_esbackup_bucketname: "elasticsearch-snapshots"
# This variable is added for the below reason -
# 1. Introduce a common variable for various clouds. In case of azure, it refers to container name, in case of aws / gcp, it refers to folder name
# 2. We want to avoid too many new variable introduction / replacement in first phase. Hence we will reuse the existing variable defined in private repo
# or other default files and just assign the value to the newly introduced common variable
# 3. After few releases, we will remove the older variables and use only the new variables across the repos
-es_backup_storage: "{{ es_azure_backup_container_name }}"
+es_backup_storage: "{{ cloud_storage_esbackup_bucketname }}"
diff --git a/ansible/roles/es-gcs-snapshot/defaults/main.yml b/ansible/roles/es-gcs-snapshot/defaults/main.yml
index 5e3cbece6f..7222b0c06b 100644
--- a/ansible/roles/es-gcs-snapshot/defaults/main.yml
+++ b/ansible/roles/es-gcs-snapshot/defaults/main.yml
@@ -1,12 +1,14 @@
snapshot_create_request_body: {
type: gcs,
settings: {
- bucket: "{{ gcs_management_bucket_name }}",
- base_path: "{{ es_backup_storage }}/{{ snapshot_base_path }}_{{ base_path_date }}"
+ bucket: "{{ cloud_storage_management_bucketname }}",
+ base_path: "{{ cloud_storage_esbackup_foldername }}/{{ snapshot_base_path }}_{{ base_path_date }}"
}
}
# Override these values
es_snapshot_host: "localhost"
snapshot_base_path: "default"
-es_backup_storage: "elasticsearch-snapshots"
\ No newline at end of file
+
+cloud_storage_esbackup_bucketname: "{{ cloud_storage_management_bucketname }}"
+cloud_storage_esbackup_foldername: "elasticsearch-snapshots"
diff --git a/ansible/roles/es-s3-snapshot/defaults/main.yml b/ansible/roles/es-s3-snapshot/defaults/main.yml
index 7ddda6ebd0..316ae512fb 100644
--- a/ansible/roles/es-s3-snapshot/defaults/main.yml
+++ b/ansible/roles/es-s3-snapshot/defaults/main.yml
@@ -1,12 +1,14 @@
snapshot_create_request_body: {
type: s3,
settings: {
- bucket: "{{ aws_management_bucket_name }}",
- base_path: "{{ es_backup_storage }}/{{ snapshot_base_path }}_{{ base_path_date }}"
+ bucket: "{{ cloud_storage_esbackup_bucketname }}",
+ base_path: "{{ cloud_storage_esbackup_foldername }}/{{ snapshot_base_path }}_{{ base_path_date }}"
}
}
# Override these values
es_snapshot_host: "localhost"
snapshot_base_path: "default"
-es_backup_storage: "elasticsearch-snapshots"
\ No newline at end of file
+
+cloud_storage_esbackup_bucketname: "{{ cloud_storage_management_bucketname }}"
+cloud_storage_esbackup_foldername: "elasticsearch-snapshots"
diff --git a/ansible/roles/influxdb_backup/tasks/main.yml b/ansible/roles/influxdb_backup/tasks/main.yml
index 11370325a0..2ddf64e950 100644
--- a/ansible/roles/influxdb_backup/tasks/main.yml
+++ b/ansible/roles/influxdb_backup/tasks/main.yml
@@ -43,6 +43,8 @@
name: gcp-cloud-storage
tasks_from: upload.yml
vars:
+ gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+ gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
gcp_bucket_name: "{{ cloud_storage_management_bucketname }}"
gcp_path: "{{ cloud_storage_influxdbbackup_foldername }}/{{ influxdb_backup_file_name }}.zip"
local_file_or_folder_path: "{{ influxdb_backup_dir }}/{{ influxdb_backup_file_name }}.zip"
diff --git a/ansible/roles/influxdb_restore/tasks/main.yml b/ansible/roles/influxdb_restore/tasks/main.yml
index 9c1f3cd436..7a2317caa0 100644
--- a/ansible/roles/influxdb_restore/tasks/main.yml
+++ b/ansible/roles/influxdb_restore/tasks/main.yml
@@ -37,6 +37,8 @@
name: gcp-cloud-storage
tasks_from: download.yml
vars:
+ gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+ gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
gcp_bucket_name: "{{ cloud_storage_influxdbbackup_bucketname }}"
gcp_path: "{{ cloud_storage_influxdbbackup_foldername }}/{{ influxdb_restore_file_name }}"
local_file_or_folder_path: "/tmp/{{ influxdb_restore_file_name }}"
diff --git a/ansible/roles/lpa-telemetry-backup-deploy/defaults/main.yml b/ansible/roles/lpa-telemetry-backup-deploy/defaults/main.yml
index b8cd06675e..08c834d1bd 100644
--- a/ansible/roles/lpa-telemetry-backup-deploy/defaults/main.yml
+++ b/ansible/roles/lpa-telemetry-backup-deploy/defaults/main.yml
@@ -2,8 +2,8 @@ analytics_user: analytics
analytics_group: analytics
analytics_user_home: /home/{{analytics_user}}
sbin_path: "{{ analytics_user_home }}/sbin"
-azure_container_name: "{{secor_azure_container_name}}"
-azure_account_key: "{{sunbird_private_storage_account_key}}"
+azure_container_name: "{{cloud_storage_telemetry_bucketname}}"
+azure_account_key: "{{cloud_private_storage_secret}}"
telemetry_ingestion_topic: "{{ env }}.telemetry.ingest"
diff --git a/ansible/roles/lpa-telemetry-backup-deploy/templates/secor.azure.j2 b/ansible/roles/lpa-telemetry-backup-deploy/templates/secor.azure.j2
index f44174563c..b2acc84e37 100644
--- a/ansible/roles/lpa-telemetry-backup-deploy/templates/secor.azure.j2
+++ b/ansible/roles/lpa-telemetry-backup-deploy/templates/secor.azure.j2
@@ -12,12 +12,12 @@ secor.azure.endpoints.protocol=https
# Microsoft Azure authentication credentials.
# https://azure.microsoft.com/en-us/documentation/articles/storage-create-storage-account
-secor.azure.account.name={{sunbird_private_storage_account_name}}
-secor.azure.account.key={{sunbird_private_storage_account_key}}
+secor.azure.account.name={{cloud_private_storage_accountname}}
+secor.azure.account.key={{cloud_private_storage_secret}}
# Microsoft Azure blob storage container name. Container is a grouping of a set
# of blobs. https://msdn.microsoft.com/en-us/library/dd135715.aspx
-secor.azure.container.name={{ azure_container_name }}
+secor.azure.container.name={{ cloud_storage_telemetry_bucketname }}
# Microsoft Azure blob storage path where files are stored within the container.
secor.azure.path={{ secor_service_name[item[0]].base_path }}
diff --git a/ansible/roles/oci-bds-spark-cluster/defaults/main.yml b/ansible/roles/oci-bds-spark-cluster/defaults/main.yml
new file mode 100644
index 0000000000..95b0b73e0d
--- /dev/null
+++ b/ansible/roles/oci-bds-spark-cluster/defaults/main.yml
@@ -0,0 +1,7 @@
+
+spark-folder: /usr/hdp/current/spark2-client
+guava_version: 19.0
+log4j_version: 2.5
+guava_url: https://repo1.maven.org/maven2/com/google/guava/guava/{{guava_version}}/guava-{{guava_version}}.jar
+log4j_core_url: https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/{{log4j_version}}/log4j-core-{{log4j_version}}.jar
+log4j_api_url: https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/{{log4j_version}}/log4j-api-{{log4j_version}}.jar
diff --git a/ansible/roles/oci-bds-spark-cluster/tasks/main.yml b/ansible/roles/oci-bds-spark-cluster/tasks/main.yml
new file mode 100644
index 0000000000..d8f4d3cc50
--- /dev/null
+++ b/ansible/roles/oci-bds-spark-cluster/tasks/main.yml
@@ -0,0 +1,13 @@
+- name: copy cluster creation script
+ template:
+ src: create-cluster.sh.j2
+ dest: /tmp/create-cluster.sh
+ mode: 0755
+ when: cluster_state == "create_cluster"
+
+- name: copy cluster deletion script
+ template:
+ src: delete-cluster.sh.j2
+ dest: /tmp/delete-cluster.sh
+ mode: 0755
+ when: cluster_state == "delete_cluster"
diff --git a/ansible/roles/oci-bds-spark-cluster/templates/create-cluster.sh.j2 b/ansible/roles/oci-bds-spark-cluster/templates/create-cluster.sh.j2
new file mode 100644
index 0000000000..161b5f3ecf
--- /dev/null
+++ b/ansible/roles/oci-bds-spark-cluster/templates/create-cluster.sh.j2
@@ -0,0 +1,320 @@
+#! /bin/bash
+# Subnet id will generate from env variable
+# Version 1 Author Nikesh Gogia and Ali Shemshadi
+
+ambari_user="${1}"
+cluster_password="${2}"
+key_alias="{{key_alias}}"
+user_id="{{user_id}}"
+subnet="{{subnet_id}}"
+compartment_id="{{compartment_id}}"
+display_name="{{display_name}}"
+workernode="{{workernode}}"
+cluster_public_key="{{public_key}}"
+
+cstate='SUCCEEDED'
+cwait=2500
+
+echo "RECEIVED ALL ENV VARIABLES"
+
+AMBARI_USER=$ambari_user
+AMBARI_PWD=$cluster_password
+
+function get_bdsid() {
+ list_param=`oci bds instance list --compartment-id $compartment_id`
+ bdsid="NULL"
+ # echo $list_param | jq '.data'
+ state="ACTIVE"
+ disname="NULL"
+ for k in $(jq '.data | keys | .[]' <<< "$list_param"); do
+ # echo $k
+ cstate=`echo $list_param | jq -r '.data['$k']["lifecycle-state"]'`
+ if [ $cstate = $state ]; then
+ disname=`echo $list_param | jq -r '.data['$k']["display-name"]'`
+ if [ $disname = $display_name ]; then
+ bdsid=`echo $list_param | jq -r '.data['$k']["id"]'`
+ fi
+
+ fi
+ echo "BDS ID"
+ echo $bdsid
+ done
+}
+
+function getLivyip() {
+
+ export bds_instance_id=$bdsid
+ bdsjson=$(oci bds instance get --bds-instance-id $bds_instance_id)
+ # echo "AMBARI URL"
+ ambari_url=`echo $bdsjson | jq -r '.data["cluster-details"]["ambari-url"]'`
+ # echo $ambari_url
+ livyip="NULL"
+ cnode="UTILITY"
+ for k in $(jq '.data["nodes"] | keys | .[]' <<< "$bdsjson"); do
+ node=`echo $bdsjson | jq -r '.data["nodes"]['$k']["node-type"]'`
+ if [ $node = "$cnode" ]; then
+ livyip=`echo $bdsjson | jq -r '.data["nodes"]['$k']["ip-address"]'`
+ fi
+ done
+ echo "LIVY IP"
+ echo $livyip
+
+}
+
+getlivyclustername() {
+ cdet=$(curl -v -u $AMBARI_USER:$AMBARI_PWD -k -X GET https://$livyip:7183/api/v1/clusters/)
+ echo $cdet
+ for k in $(jq '.items | keys | .[]' <<< "$cdet"); do
+ # echo $k
+ cluster_name=`echo $cdet | jq -r '.items['$k']["Clusters"]["cluster_name"]'`
+ echo $cluster_name
+ done
+ echo "CLUSTER NAME"
+
+}
+
+function get_apidetails() {
+
+ export bds_instance_id=$bdsid
+
+ listapijson=$(oci bds bds-api-key list --bds-instance-id $bds_instance_id)
+
+ #echo $listapijson | jq '.data[1]["key-alias"]'
+ id="NULL"
+ ctype="ACTIVE"
+ for k in $(jq '.data | keys | .[]' <<< "$listapijson"); do
+ type=`echo $listapijson | jq -r '.data['$k']["lifecycle-state"]'`
+ if [ $type = "$ctype" ]; then
+ id=`echo $listapijson | jq -r '.data['$k']["id"]'`
+ fi
+ done
+
+ echo $id
+
+ export api_key_id=$id
+
+ list_api=`oci bds bds-api-key get --api-key-id $api_key_id --bds-instance-id $bds_instance_id `
+
+ #echo $list_api | jq '.data'
+
+ data=`echo $list_api | jq '.data'`
+ echo "API DETAILS"
+ echo $data
+ region=`echo $list_api | jq -r '.data["default-region"]'`
+ fingerprint=`echo $list_api | jq -r '.data["fingerprint"]'`
+ keyalias=`echo $list_api | jq -r '.data["key-alias"]'`
+ lifecyc=`echo $list_api | jq -r '.data["lifecycle-state"]'`
+ tm=`echo $list_api | jq -r '.data["time-created"]'`
+ usid=`echo $list_api | jq -r '.data["user-id"]'`
+ tenid=`echo $list_api | jq -r '.data["tenant-id"]'`
+ pemfilepath=`echo $list_api | jq -r '.data["pemfilepath"]'`
+
+}
+
+function update_bds_config(){
+ #change below variables for your cluster
+ CONFIG_FILE_TO_UPDATE=""
+
+ #Used when for restarting components after config update
+ #Wait time before we poll for restart status. Default 30 seconds. Meaning, We poll for restart status every 30 seconds
+ WAIT_TIME_IN_SEC=30
+
+ #No of tries before we give up on the restart status. Default 20. With default WAIT_TIME_IN_SEC as 30, At max we wait for 10(20*30=600 seconds) minutes before we give up.
+ RETRY_COUNT=20
+
+ #INTERNAL USE ONLY
+ propObj=""
+
+ get_apidetails
+ getUtilityNodesIps=$livyip
+ getlivyclustername
+ echo $getUtilityNodesIps
+ getClusterName=$cluster_name
+ for utilityNodeIp in $getUtilityNodesIps
+ do
+ echo "Current utility node ip: $utilityNodeIp"
+ str1=$(nslookup $utilityNodeIp | awk -v var=$utilityNodeIp '/name =/{print var "\t", $4}')
+ CONFIG_FILE_TO_UPDATE="core-site" #this is the file we're updating in this example
+ propObj=$(get_property_json)
+ echo $propObj
+ echo "calling add properties"
+
+ #update key value pairs. Multiple key value pairs can be updated before doing update_ambari_config
+ add_properties "fs.oci.client.auth.fingerprint" $fingerprint
+ add_properties "fs.oci.client.auth.passphrase" $cluster_password
+ add_properties "fs.oci.client.auth.pemfilepath" $pemfilepath
+ add_properties "fs.oci.client.auth.tenantId" $tenid
+ add_properties "fs.oci.client.auth.userId" $usid
+ add_properties "fs.oci.client.regionCodeOrId" $region
+ add_properties "fs.s3.buffer.dir" /tmp
+ #Update it to ambari
+ echo "updating ambari config"
+ update_ambari_config
+
+ echo "restarting all required components"
+ restart_required_components
+
+ done
+
+}
+
+
+#Method to collect the current config
+function get_property_json(){
+ allConfs=$(curl -v -u $AMBARI_USER:$AMBARI_PWD -k -X GET https://$utilityNodeIp:7183/api/v1/clusters/$getClusterName?fields=Clusters/desired_configs) #to get all the configs
+ currVersionLoc=".Clusters.desired_configs.\"$CONFIG_FILE_TO_UPDATE\".tag" #fetching current version for property
+ propVersion=$(echo $allConfs | jq $currVersionLoc | tr -d '"')
+ propJson=$(curl -u $AMBARI_USER:$AMBARI_PWD -H "X-Requested-By: ambari" -k -X GET "https://$utilityNodeIp:7183/api/v1/clusters/$getClusterName/configurations?type=$CONFIG_FILE_TO_UPDATE&tag=$propVersion") #fetch property json
+ propLoc=".items[].properties"
+ propKeyVal=$(echo $propJson | jq $propLoc)
+ propObj="{\"properties\":$propKeyVal}"
+ echo $propObj
+}
+
+#Method to add/update key value pair to existing config
+function add_properties(){
+ echo $1 $2
+ echo $propObj
+ propObj=$(echo $propObj | jq '.properties += { "'$1'": "'$2'" }')
+ echo $propObj
+}
+
+#Method to update config in ambari
+function update_ambari_config(){
+ parseableAddedProp=$(echo $propObj | jq '.properties')
+ echo $parseableAddedProp
+ timestamp=$(date +%s)
+ newVersion="version$timestamp"
+ finalJson='[{"Clusters":{"desired_config":[{"type":"'$CONFIG_FILE_TO_UPDATE'","tag":"'$newVersion'","properties":'$parseableAddedProp'}]}}]'
+ echo "CALING AMABRI API"
+ response_body_amb=$(curl -u $AMBARI_USER:$AMBARI_PWD -H "X-Requested-By: ambari" -k -X PUT -d "$finalJson" "https://$utilityNodeIp:7183/api/v1/clusters/$getClusterName")
+ echo $response_body_amb
+ echo "DONE AMABRI API"
+}
+
+#Method to restart required components
+function restart_required_components(){
+ echo "restarting all required components"
+ response_body=$(curl -u $AMBARI_USER:$AMBARI_PWD -H "X-Requested-By: ambari" -k -X POST -d '{"RequestInfo":{"command":"RESTART","context":"Restart all required services from bootstrap script","operation_level":"host_component"},"Requests/resource_filters":[{"hosts_predicate":"HostRoles/stale_configs=true&HostRoles/cluster_name='$getClusterName'"}]}' "https://$utilityNodeIp:7183/api/v1/clusters/$getClusterName/requests")
+
+ echo "printing response_body: $response_body"
+
+ idLoc=".Requests.id"
+ requestId=$(echo $response_body | jq $idLoc)
+ echo "request id is : $requestId"
+
+ current_count=0
+ while [[ $current_count -lt $RETRY_COUNT ]];
+ do
+ current_count=$((current_count+1))
+ response=$(curl -v -u $AMBARI_USER:$AMBARI_PWD -k -X GET https://$utilityNodeIp:7183/api/v1/clusters/$getClusterName/requests/$requestId)
+ request_status=$(echo $response | jq -r ".Requests.request_status")
+ echo "printing request_status: $request_status"
+ if [[ $request_status == "IN_PROGRESS" ]] || [[ $request_status == "PENDING" ]]; then
+ echo "current_count is : $current_count"
+ sleep $WAIT_TIME_IN_SEC
+ elif [[ $request_status == "COMPLETED" ]]; then
+ echo "Restart successful"
+ break
+ fi
+ done
+}
+
+function create_api(){
+ export bds_instance_id=$bdsid
+ export key_alias=$key_alias # https://docs.cloud.oracle.com/en-us/iaas/tools/oci-cli/latest/oci_cli_docs/cmdref/bds/bds-api-key/create.html#cmdoption-key-alias
+ export passphrase=$b64p # https://docs.cloud.oracle.com/en-us/iaas/tools/oci-cli/latest/oci_cli_docs/cmdref/bds/bds-api-key/create.html#cmdoption-passphrase
+ export user_id=$user_id
+ capi='SUCCEEDED'
+ oci bds bds-api-key create --bds-instance-id $bds_instance_id --key-alias $key_alias --passphrase $passphrase --user-id $user_id --wait-for-state $capi --max-wait-seconds $cwait
+}
+
+
+# Below is tenancy
+
+function create_cluster() {
+
+ export compartment_id=$compartment_id
+
+ master=1
+ utility=1
+
+ worker=$workernode # This has to be replaced with Jenkins Paramter
+
+ # Begin script in case all parameters are correct
+ echo "Generating json woth $master master ndoes $utility utility nodes and $worker worker nodes"
+ json="["
+
+ for i in `seq 1 $master`
+ do
+ json="$json{\"blockVolumeSizeInGBs\": 1000,\"nodeType\": \"MASTER\",\"shape\": \"VM.Standard.E4.Flex\",\"shapeConfig\": { \"memoryInGBs\": {{head_node_memory}}, \"ocpus\": {{head_node_cpu}}},\"subnetId\": \"$subnet\" }"
+ done
+
+ for i in `seq 1 $utility`
+ do
+ json="$json,{\"blockVolumeSizeInGBs\": 1000,\"nodeType\": \"UTILITY\",\"shape\": \"VM.Standard.E4.Flex\",\"shapeConfig\": { \"memoryInGBs\": {{head_node_memory}}, \"ocpus\": {{head_node_cpu}}},\"subnetId\": \"$subnet\" }"
+ done
+
+ for i in `seq 1 $worker`
+ do
+ json="$json,{\"blockVolumeSizeInGBs\": 1000,\"nodeType\": \"WORKER\",\"shape\": \"VM.Standard.E4.Flex\",\"shapeConfig\": { \"memoryInGBs\": {{worker_node_memory}}, \"ocpus\": {{worker_node_cpu}}},\"subnetId\": \"$subnet\" }"
+ done
+
+ json="$json]"
+ printf "$json" > "nodes.json"
+ echo "File successfully generated and saved as nodes.json"
+
+ echo "TRIGGERED CREATING THE BDS CLUSTER"
+
+ export cluster_public_key=$public_key
+ export cluster_version="ODH2_0"
+ export display_name=$display_name
+ export is_high_availability='false'
+ export is_secure='false'
+ cmd="oci bds instance create --cluster-admin-password '$b64p' --cluster-public-key '$cluster_public_key' --cluster-version '$cluster_version' --compartment-id '$compartment_id' --display-name '$display_name' --is-high-availability $is_high_availability --is-secure $is_secure --wait-for-state $cstate --max-wait-seconds $cwait --nodes file://nodes.json "
+ #echo $cmd
+ create_response=$(eval "$cmd")
+ echo "CLUSTER CREATED SUCCESSFULLY"
+}
+
+function replace_host() {
+ echo "REPLACE THE HOSTS"
+ echo "" >> {{inventory_dir}}/hosts
+ echo "[bds-livy-node]" >> {{inventory_dir}}/hosts
+ echo "$livyip ansible_ssh_user=opc" >> {{inventory_dir}}/hosts
+ echo "" >> {{inventory_dir}}/hosts
+
+}
+
+# MAIN TO START
+
+b64p=`echo -n $cluster_password | base64`
+echo $b64p
+echo $compartment_id
+
+echo "CREATING BDS CLUSTER"
+
+create_cluster
+
+echo "FETCHING BDS ID"
+
+get_bdsid # This sets BDS ID
+
+echo "GET LIVY-AMBARI IP"
+
+getLivyip # This will be ambari ip also
+
+replace_host
+
+echo "CREATE OBJECT STORAGE API KEY"
+
+create_api
+
+echo "UPDATE BDS AMBARI CONFIG"
+
+get_apidetails
+
+update_bds_config
+
+echo "BDS Config Completed and Ambari Restarted"
+
diff --git a/ansible/roles/oci-bds-spark-cluster/templates/delete-cluster.sh.j2 b/ansible/roles/oci-bds-spark-cluster/templates/delete-cluster.sh.j2
new file mode 100755
index 0000000000..3e3df78b33
--- /dev/null
+++ b/ansible/roles/oci-bds-spark-cluster/templates/delete-cluster.sh.j2
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Version 1 - Author Nikesh Gogia nikesh.g.gogia@oracle.com
+
+compartment_id="{{compartment_id}}"
+display_name="{{display_name}}"
+
+
+echo "DELETE STARTED"
+echo $display_name
+echo $compartment_id
+
+
+function get_bdsid() {
+ list_param=`oci bds instance list --compartment-id $compartment_id`
+ bdsid="NULL"
+ # echo $list_param | jq '.data'
+ state="ACTIVE"
+ disname="NULL"
+ for k in $(jq '.data | keys | .[]' <<< "$list_param"); do
+ # echo $k
+ cstate=`echo $list_param | jq -r '.data['$k']["lifecycle-state"]'`
+ if [ $cstate = $state ]; then
+ disname=`echo $list_param | jq -r '.data['$k']["display-name"]'`
+ if [ $disname = $display_name ]; then
+ bdsid=`echo $list_param | jq -r '.data['$k']["id"]'`
+ fi
+
+ fi
+ echo $bdsid
+ done
+}
+
+get_bdsid
+
+yes Y | oci bds instance delete --bds-instance-id $bdsid
diff --git a/ansible/roles/oci-cli/defaults/main.yml b/ansible/roles/oci-cli/defaults/main.yml
new file mode 100644
index 0000000000..147a2e03f1
--- /dev/null
+++ b/ansible/roles/oci-cli/defaults/main.yml
@@ -0,0 +1 @@
+oci_cli_url: https://github.com/oracle/oci-cli/releases/download/v3.22.0/oci-cli-3.22.0-Ubuntu-18.04-Offline.zip
\ No newline at end of file
diff --git a/ansible/roles/oci-cli/tasks/main.yml b/ansible/roles/oci-cli/tasks/main.yml
new file mode 100644
index 0000000000..389a9e8235
--- /dev/null
+++ b/ansible/roles/oci-cli/tasks/main.yml
@@ -0,0 +1,24 @@
+---
+- name: Download the installation file
+ get_url:
+ url: "{{ oci_cli_url }}"
+ dest: /tmp/ocicli.zip
+
+- name: Installing unzip
+ apt:
+ name: "{{item}}"
+ state: latest
+ with_items:
+ - zip
+ - unzip
+
+- name: Unzip the installer
+ unarchive:
+ src: /tmp/ocicli.zip
+ dest: /tmp/
+ remote_src: yes
+
+- name: install oci cli
+ shell: ./oci-cli-installation/install.sh --install-dir {{ analytics_user_home }} --exec-dir {{ analytics_user_home }} --script-dir {{ analytics_user_home }} --accept-all-defaults
+ args:
+ chdir: /tmp/
diff --git a/ansible/roles/oci-cloud-storage/defaults/main.yml b/ansible/roles/oci-cloud-storage/defaults/main.yml
new file mode 100644
index 0000000000..72727de167
--- /dev/null
+++ b/ansible/roles/oci-cloud-storage/defaults/main.yml
@@ -0,0 +1,3 @@
+oss_bucket_name: ""
+oss_path: ""
+local_file_or_folder_path: ""
diff --git a/ansible/roles/oci-cloud-storage/tasks/delete-folder.yml b/ansible/roles/oci-cloud-storage/tasks/delete-folder.yml
new file mode 100644
index 0000000000..6ed4e6b8b4
--- /dev/null
+++ b/ansible/roles/oci-cloud-storage/tasks/delete-folder.yml
@@ -0,0 +1,5 @@
+---
+- name: delete files and folders recursively
+ shell: "oci os object bulk-delete -ns {{oss_namespace}} -bn {{oss_bucket_name}} --prefix {{oss_path}} --force"
+ async: 3600
+ poll: 10
diff --git a/ansible/roles/oci-cloud-storage/tasks/delete.yml b/ansible/roles/oci-cloud-storage/tasks/delete.yml
new file mode 100644
index 0000000000..65d18843ca
--- /dev/null
+++ b/ansible/roles/oci-cloud-storage/tasks/delete.yml
@@ -0,0 +1,7 @@
+- name: Ensure oci oss bucket exists
+ command: oci os bucket get --name {{ oss_bucket_name }}
+
+- name: Upload to oci oss bucket
+ command: oci os object delete -bn {{ oss_bucket_name }} --name {{ oss_path }} --force
+ async: 3600
+ poll: 10
\ No newline at end of file
diff --git a/ansible/roles/oci-cloud-storage/tasks/download.yml b/ansible/roles/oci-cloud-storage/tasks/download.yml
new file mode 100644
index 0000000000..bb32e9ed93
--- /dev/null
+++ b/ansible/roles/oci-cloud-storage/tasks/download.yml
@@ -0,0 +1,7 @@
+- name: Ensure oci oss bucket exists
+ command: oci os bucket get --name {{ oss_bucket_name }}
+
+- name: download files from oci oss bucket
+ command: oci os object get -bn {{ oss_bucket_name }} --name {{ oss_object_name }} --file {{ local_file_or_folder_path }}
+ async: 3600
+ poll: 10
diff --git a/ansible/roles/oci-cloud-storage/tasks/main.yml b/ansible/roles/oci-cloud-storage/tasks/main.yml
new file mode 100644
index 0000000000..6f9dca6b63
--- /dev/null
+++ b/ansible/roles/oci-cloud-storage/tasks/main.yml
@@ -0,0 +1,18 @@
+---
+- name: delete files from oci oss bucket
+ include: delete.yml
+
+- name: delete folders from oci oss bucket recursively
+ include: delete-folder.yml
+
+
+- name: download file from oss
+ include: download.yml
+
+- name: upload files from a local to oci oss
+ include: upload.yml
+
+- name: upload files and folder from local directory to oci oss
+ include: upload-folder.yml
+
+
diff --git a/ansible/roles/oci-cloud-storage/tasks/upload-folder.yml b/ansible/roles/oci-cloud-storage/tasks/upload-folder.yml
new file mode 100644
index 0000000000..6e4d06562c
--- /dev/null
+++ b/ansible/roles/oci-cloud-storage/tasks/upload-folder.yml
@@ -0,0 +1,8 @@
+---
+- name: Ensure oci oss bucket exists
+ command: oci os bucket get --name {{ oss_bucket_name }}
+
+- name: Upload folder to oci oss bucket
+ command: oci os object bulk-upload -bn {{ oss_bucket_name }} --prefix {{ oss_path }} --src-dir {{ local_file_or_folder_path }} --content-type auto
+ async: 3600
+ poll: 10
diff --git a/ansible/roles/oci-cloud-storage/tasks/upload.yml b/ansible/roles/oci-cloud-storage/tasks/upload.yml
new file mode 100644
index 0000000000..2771da5771
--- /dev/null
+++ b/ansible/roles/oci-cloud-storage/tasks/upload.yml
@@ -0,0 +1,8 @@
+---
+- name: Ensure oci oss bucket exists
+ command: oci os bucket get --name {{ oss_bucket_name }}
+
+- name: Upload to oci oss bucket
+ command: oci os object put -bn {{ oss_bucket_name }} --name {{ oss_path }} --file {{ local_file_or_folder_path }} --content-type auto --force
+ async: 3600
+ poll: 10
diff --git a/ansible/roles/portal-dashboard/tasks/main.yml b/ansible/roles/portal-dashboard/tasks/main.yml
index adfbd76b39..7c53a72307 100644
--- a/ansible/roles/portal-dashboard/tasks/main.yml
+++ b/ansible/roles/portal-dashboard/tasks/main.yml
@@ -12,7 +12,7 @@
pip:
name: "{{library_path}}"
virtualenv: "{{ virtualenv_path }}"
- virtualenv_python: "python3.6"
+ # virtualenv_python: "python3.6"
tags:
- common
diff --git a/ansible/roles/postgres-db-update/tasks/main.yml b/ansible/roles/postgres-db-update/tasks/main.yml
index 631299b253..b76eda5641 100644
--- a/ansible/roles/postgres-db-update/tasks/main.yml
+++ b/ansible/roles/postgres-db-update/tasks/main.yml
@@ -7,48 +7,48 @@
- name: Ensure database is created
postgresql_db: name="{{ postgres.db_name }}" \
- login_host="{{ postgres.db_url }}" \
- port="{{ postgres.db_port }}" \
- login_user="{{ postgres.db_admin_user }}" \
- login_password="{{ postgres.db_admin_password }}" \
- encoding='UTF-8' \
- state=present
+ login_host="{{ postgres.db_url }}" \
+ port="{{ postgres.db_port }}" \
+ login_user="{{ postgres.db_admin_user }}" \
+ login_password="{{ postgres.db_admin_password }}" \
+ encoding='UTF-8' \
+ state=present
tags: createdb
- name: Ensure database for superset is created
postgresql_db: name="superset" \
- login_host="{{ postgres.db_url }}" \
- port="{{ postgres.db_port }}" \
- login_user="{{ postgres.db_admin_user }}" \
- login_password="{{ postgres.db_admin_password }}" \
- encoding='UTF-8' \
- state=present
+ login_host="{{ postgres.db_url }}" \
+ port="{{ postgres.db_port }}" \
+ login_user="{{ postgres.db_admin_user }}" \
+ login_password="{{ postgres.db_admin_password }}" \
+ encoding='UTF-8' \
+ state=present
tags: createdb
- name: Ensure user has access to the database
postgresql_user: name="{{ postgres.db_username }}" \
- password="{{ postgres.db_password }}" \
- no_password_changes=true \
- priv=ALL \
- state=present \
- login_host="{{ postgres.db_url }}" \
- port="{{ postgres.db_port }}" \
- login_user="{{ postgres.db_admin_user }}" \
- login_password="{{ postgres.db_admin_password }}" \
- db="{{ postgres.db_name }}"
+ password="{{ postgres.db_password }}" \
+ no_password_changes=true \
+ priv=ALL \
+ state=present \
+ login_host="{{ postgres.db_url }}" \
+ port="{{ postgres.db_port }}" \
+ login_user="{{ postgres.db_admin_user }}" \
+ login_password="{{ postgres.db_admin_password }}" \
+ db="{{ postgres.db_name }}"
tags: createuser
- name: Ensure user has access to the superset database
postgresql_user: name="{{ postgres.db_username }}" \
- password="{{ postgres.db_password }}" \
- no_password_changes=true \
- priv=ALL \
- state=present \
- login_host="{{ postgres.db_url }}" \
- port="{{ postgres.db_port }}" \
- login_user="{{ postgres.db_admin_user }}" \
- login_password="{{ postgres.db_admin_password }}" \
- db="superset"
+ password="{{ postgres.db_password }}" \
+ no_password_changes=true \
+ priv=ALL \
+ state=present \
+ login_host="{{ postgres.db_url }}" \
+ port="{{ postgres.db_port }}" \
+ login_user="{{ postgres.db_admin_user }}" \
+ login_password="{{ postgres.db_admin_password }}" \
+ db="superset"
tags: createuser
- name: create {{ env }}_consumer-channel-mapping
@@ -74,7 +74,7 @@
- name: update {{ env }}_report_config
shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE {{ env }}_report_config ADD COLUMN IF NOT EXISTS batch_number INTEGER"
run_once: true
- tags: update
+ tags: update
- name: create report
shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS report(reportid varchar(40) NOT NULL PRIMARY KEY, title text NOT NULL, description text NOT NULL, authorizedroles jsonb NOT NULL, status varchar(8) NOT NULL CHECK (status IN ('live', 'draft', 'retired')) DEFAULT 'draft', type varchar(8) NOT NULL CHECK (type in ('public', 'private')) DEFAULT 'private', reportaccessurl text NOT NULL UNIQUE, createdon timestamptz NOT NULL DEFAULT now(), updatedon timestamptz NOT NULL DEFAULT now(), createdby varchar(50) NOT NULL, reportconfig jsonb NOT NULL, templateurl text, slug varchar(10) NOT NULL, reportgenerateddate timestamptz NOT NULL DEFAULT now(), reportduration jsonb NOT NULL DEFAULT jsonb_build_object('startDate', now()::timestamptz, 'endDate', now()::timestamptz), tags jsonb NOT NULL, updatefrequency text NOT NULL)"
@@ -114,7 +114,7 @@
- name: create {{ env }}_experiment_definition
shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_experiment_definition (exp_id VARCHAR(50), created_by VARCHAR(50), created_on TIMESTAMP, criteria VARCHAR(100), exp_data VARCHAR(300), exp_description VARCHAR(200), exp_name VARCHAR(50), stats VARCHAR(300), status VARCHAR(50), status_message VARCHAR(50), updated_by VARCHAR(50), updated_on TIMESTAMP, PRIMARY KEY(exp_id))"
run_once: true
- tags: create
+ tags: create
- name: create {{ env }}_dataset_metadata
shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_dataset_metadata(dataset_id VARCHAR(50), dataset_sub_id VARCHAR(150), dataset_config json, visibility VARCHAR(50), dataset_type VARCHAR(50), version VARCHAR(10), authorized_roles text[], available_from TIMESTAMP, sample_request text, sample_response text, validation_json json, druid_query json, limits json, supported_formats text[], exhaust_type VARCHAR(50), PRIMARY KEY (dataset_id, dataset_sub_id))"
@@ -139,7 +139,7 @@
- name: update report
shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE report ADD COLUMN IF NOT EXISTS report_type varchar(8) NOT NULL DEFAULT 'report'"
run_once: true
- tags: update
+ tags: update
- name: update report
shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE report ADD COLUMN IF NOT EXISTS visibilityflags jsonb"
@@ -164,4 +164,9 @@
- name: update report
shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE report ALTER COLUMN type TYPE varchar(10)"
run_once: true
- tags: update
\ No newline at end of file
+ tags: update
+
+- name: update encryption_key in {{ env }}_job_request
+ shell: PGPASSWORD="{{postgres.db_password}}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE {{ env }}_job_request ALTER encryption_key TYPE varchar(500)"
+ run_once: true
+ tags: update
diff --git a/ansible/roles/postgres-managed-service-restore/tasks/main.yml b/ansible/roles/postgres-managed-service-restore/tasks/main.yml
index db903740cd..c46e43e647 100644
--- a/ansible/roles/postgres-managed-service-restore/tasks/main.yml
+++ b/ansible/roles/postgres-managed-service-restore/tasks/main.yml
@@ -39,6 +39,8 @@
name: gcp-cloud-storage
tasks_from: download.yml
vars:
+ gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+ gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
gcp_bucket_name: "{{ cloud_storage_dppostgresqlbackup_bucketname }}"
gcp_path: "{{ cloud_storage_dppostgresqlbackup_foldername }}/{{ postgres_backup_filename }}"
local_file_or_folder_path: "{{ postgres_restore_dir }}/{{ postgres_backup_filepath }}"
diff --git a/ansible/roles/postgres-managed-service/tasks/main.yml b/ansible/roles/postgres-managed-service/tasks/main.yml
index 1a8c69657f..c4189f2a08 100644
--- a/ansible/roles/postgres-managed-service/tasks/main.yml
+++ b/ansible/roles/postgres-managed-service/tasks/main.yml
@@ -59,6 +59,8 @@
name: gcp-cloud-storage
tasks_from: upload.yml
vars:
+ gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+ gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
gcp_bucket_name: "{{ cloud_storage_dppostgresqlbackup_bucketname }}"
gcp_path: "{{ cloud_storage_dppostgresqlbackup_foldername }}/{{ postgresql_backup_gzip_file_name }}.zip"
local_file_or_folder_path: "{{ postgresql_backup_gzip_file_path }}"
diff --git a/ansible/roles/postgresql-backup/tasks/main.yml b/ansible/roles/postgresql-backup/tasks/main.yml
index ddff9d62bd..4be4bda8db 100755
--- a/ansible/roles/postgresql-backup/tasks/main.yml
+++ b/ansible/roles/postgresql-backup/tasks/main.yml
@@ -44,6 +44,8 @@
name: gcp-cloud-storage
tasks_from: upload.yml
vars:
+ gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+ gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
gcp_bucket_name: "{{ cloud_storage_dppostgresqlbackup_bucketname }}"
gcp_path: "{{ cloud_storage_dppostgresqlbackup_foldername }}/{{ postgresql_backup_gzip_file_name }}"
local_file_or_folder_path: "{{ postgresql_backup_gzip_file_path }}"
diff --git a/ansible/roles/postgresql-restore/tasks/main.yml b/ansible/roles/postgresql-restore/tasks/main.yml
index 275061e9f2..7635f7e741 100755
--- a/ansible/roles/postgresql-restore/tasks/main.yml
+++ b/ansible/roles/postgresql-restore/tasks/main.yml
@@ -34,6 +34,8 @@
name: gcp-cloud-storage
tasks_from: download.yml
vars:
+ gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+ gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
gcp_bucket_name: "{{ cloud_storage_dppostgresqlbackup_bucketname }}"
gcp_path: "{{ cloud_storage_dppostgresqlbackup_foldername }}/{{ postgresql_restore_gzip_file_name }}"
local_file_or_folder_path: "{{ postgresql_restore_gzip_file_path }}"
diff --git a/ansible/roles/provision-azure-spark-cluster/tasks/main.yml b/ansible/roles/provision-azure-spark-cluster/tasks/main.yml
index c126dc8f3b..913837c651 100644
--- a/ansible/roles/provision-azure-spark-cluster/tasks/main.yml
+++ b/ansible/roles/provision-azure-spark-cluster/tasks/main.yml
@@ -1,11 +1,18 @@
- name: Adding azure blob variable to spark env file
lineinfile:
path: "{{spark_folder}}/conf/spark-env.sh"
- line: '{{item.var}}={{item.value}}'
+ line: "{{item.var}}={{item.value}}"
regexp: "{{ item.var }}.*"
with_items:
- - {var: 'azure_storage_key', value: '{{ azure_private_storage_account_name }}'}
- - {var: 'azure_storage_secret', value: '{{ azure_private_storage_account_key }}'}
+ - {
+ var: "azure_storage_key",
+ value: "{{ cloud_private_storage_accountname }}",
+ }
+ - {
+ var: "azure_storage_secret",
+ value: "{{ cloud_private_storage_secret }}",
+ }
+
no_log: true
when: cloud_service_provider == "azure"
@@ -15,18 +22,18 @@
path: "{{ spark_folder }}/jars/{{item.var}}-{{item.value}}.jar"
state: absent
with_items:
- - {var: 'guava', value: '{{ guava_default_jre_version }}'}
- - {var: 'guice', value: '{{ guice_default_version }}'}
- - {var: 'guice-servlet', value: '{{ guice_default_version }}'}
-
+ - { var: "guava", value: "{{ guava_default_jre_version }}" }
+ - { var: "guice", value: "{{ guice_default_version }}" }
+ - { var: "guice-servlet", value: "{{ guice_default_version }}" }
+
- name: Download guava_jre_url and copy to Spark jars folder
become: yes
get_url: url={{ guava_jre_url }} dest={{ spark_folder }}/jars/guava-{{guava_jre_version}}.jar timeout=1000 force=no
-
+
- name: Download log4j api and copy to Spark jars folder
become: yes
get_url: url={{ log4j_api_url }} dest={{ spark_folder }}/jars/log4j-api-{{log4j_version}}.jar timeout=1000 force=no
-
+
- name: Download log4j core and copy to Spark jars folder
become: yes
get_url: url={{ log4j_core_url }} dest={{ spark_folder }}/jars/log4j-core-{{log4j_version}}.jar timeout=1000 force=no
diff --git a/ansible/roles/provision-oci-spark-cluster/defaults/main.yml b/ansible/roles/provision-oci-spark-cluster/defaults/main.yml
new file mode 100644
index 0000000000..777554f8ba
--- /dev/null
+++ b/ansible/roles/provision-oci-spark-cluster/defaults/main.yml
@@ -0,0 +1,44 @@
+
+bucket: "telemetry-data-store"
+model_version: "2.0"
+
+spark_folder: /usr/odh/2.0.1/spark
+
+# delete
+guava_default_version: 14.0.1
+guava_default_jre_version_1: 26.0-jre
+guava_default_jre_version_2: 27.0-jre
+guice_default_version: 4.2.2
+
+# add
+guava_version: 19.0
+log4j_version: 2.16.0
+spark_redis_version: 2.5.0
+guava_jre_version: 24.1.1-jre
+jedis_version: 3.2.0
+zip4j_version: 2.6.2
+guice_version: 3.0
+
+jets3t_version: 0.9.7
+hadoop_aws_version: 2.7.3
+java_xmlbuilder_version: 1.1
+cassandra_connector_version: 3.2.0
+commons_pool_version: 2.0
+
+guava_url: https://repo1.maven.org/maven2/com/google/guava/guava/{{guava_version}}/guava-{{guava_version}}.jar
+guava_jre_url: https://repo1.maven.org/maven2/com/google/guava/guava/{{guava_jre_version}}/guava-{{guava_jre_version}}.jar
+log4j_core_url: https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/{{log4j_version}}/log4j-core-{{log4j_version}}.jar
+log4j_api_url: https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/{{log4j_version}}/log4j-api-{{log4j_version}}.jar
+spark_redis_url: https://repo1.maven.org/maven2/com/redislabs/spark-redis_2.12/{{spark_redis_version}}/spark-redis_2.12-{{spark_redis_version}}.jar
+jedis_url: https://repo1.maven.org/maven2/redis/clients/jedis/{{jedis_version}}/jedis-{{jedis_version}}.jar
+zip4j_url: https://repo1.maven.org/maven2/net/lingala/zip4j/zip4j/{{zip4j_version}}/zip4j-{{zip4j_version}}.jar
+guice_url: https://repo1.maven.org/maven2/com/google/inject/guice/{{guice_version}}/guice-{{guice_version}}.jar
+guice_servlet_url: https://repo1.maven.org/maven2/com/google/inject/extensions/guice-servlet/{{guice_version}}/guice-servlet-{{guice_version}}.jar
+
+# jets3t_url: https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/{{jets3t_version}}/jets3t-{{jets3t_version}}.jar
+jets3t_url: https://repo1.maven.org/maven2/org/jets3t/jets3t/{{jets3t_version}}/jets3t-{{jets3t_version}}.jar
+hadoop_aws_url: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/{{hadoop_aws_version}}/hadoop-aws-{{hadoop_aws_version}}.jar
+java_xmlbuilder_url: https://repo1.maven.org/maven2/com/jamesmurty/utils/java-xmlbuilder/{{java_xmlbuilder_version}}/java-xmlbuilder-{{java_xmlbuilder_version}}.jar
+common_pool_url: "https://repo1.maven.org/maven2/org/apache/commons/commons-pool2/{{commons_pool_version}}/commons-pool2-{{commons_pool_version}}.jar"
+spark_cassandra_connector_assembly_url: "https://repo1.maven.org/maven2/com/datastax/spark/spark-cassandra-connector-assembly_2.12/{{cassandra_connector_version}}/spark-cassandra-connector-assembly_2.12-{{cassandra_connector_version}}.jar"
+
diff --git a/ansible/roles/provision-oci-spark-cluster/tasks/main.yml b/ansible/roles/provision-oci-spark-cluster/tasks/main.yml
new file mode 100644
index 0000000000..4c2302d168
--- /dev/null
+++ b/ansible/roles/provision-oci-spark-cluster/tasks/main.yml
@@ -0,0 +1,99 @@
+# - name: Adding azure blob variable to spark env file
+ # lineinfile:
+ # path: "{{spark_folder}}/conf/spark-env.sh"
+ # line: '{{item.var}}={{item.value}}'
+ # regexp: "{{ item.var }}.*"
+ # with_items:
+ # - {var: 'azure_storage_key', value: '{{ azure_private_storage_account_name }}'}
+ # - {var: 'azure_storage_secret', value: '{{ azure_private_storage_account_key }}'}
+ # no_log: true
+ # when: cloud_service_provider == "azure"
+
+- name: Remove guava-jre, guice default jars
+ become: yes
+ file:
+ path: "{{ spark_folder }}/jars/{{item.var}}-{{item.value}}.jar"
+ state: absent
+ with_items:
+ - {var: 'guava', value: '{{ guava_default_version }}'}
+ - {var: 'guava', value: '{{ guava_default_jre_version_1 }}'}
+ - {var: 'guava', value: '{{ guava_default_jre_version_2 }}'}
+ - {var: 'guice', value: '{{ guice_default_version }}'}
+ - {var: 'guice-servlet', value: '{{ guice_default_version }}'}
+
+- name: Download guava and copy to Spark jars folder
+ become: yes
+ get_url: url={{ guava_url }} dest={{ spark_folder }}/jars/guava-{{guava_version}}.jar timeout=1000 force=no
+
+- name: Download guava_jre_url and copy to Spark jars folder
+ become: yes
+ get_url: url={{ guava_jre_url }} dest={{ spark_folder }}/jars/guava-{{guava_jre_version}}.jar timeout=1000 force=no
+
+- name: Download log4j api and copy to Spark jars folder
+ become: yes
+ get_url: url={{ log4j_api_url }} dest={{ spark_folder }}/jars/log4j-api-{{log4j_version}}.jar timeout=1000 force=no
+
+- name: Download log4j core and copy to Spark jars folder
+ become: yes
+ get_url: url={{ log4j_core_url }} dest={{ spark_folder }}/jars/log4j-core-{{log4j_version}}.jar timeout=1000 force=no
+
+- name: Download spark-redis and copy to Spark jars folder
+ become: yes
+ get_url: url={{ spark_redis_url }} dest={{ spark_folder }}/jars/spark-redis_2.12-{{spark_redis_version}}.jar timeout=1000 force=no
+
+- name: Download jedis and copy to Spark jars folder
+ become: yes
+ get_url: url={{ jedis_url }} dest={{ spark_folder }}/jars/jedis-{{jedis_version}}.jar timeout=1000 force=no
+
+- name: Download zip4j and copy to Spark jars folder
+ become: yes
+ get_url: url={{ zip4j_url }} dest={{ spark_folder }}/jars/zip4j-{{zip4j_version}}.jar timeout=1000 force=no
+
+- name: Download guice and copy to Spark jars folder
+ become: yes
+ get_url: url={{ guice_url }} dest={{ spark_folder }}/jars/guice-{{guice_version}}.jar timeout=1000 force=no
+
+- name: Download guice-servlet and copy to Spark jars folder
+ become: yes
+ get_url: url={{ guice_servlet_url }} dest={{ spark_folder }}/jars/guice-servlet-{{guice_version}}.jar timeout=1000 force=no
+
+- name: Download jets3t and copy to Spark jars folder
+ become: yes
+ get_url: url={{ jets3t_url }} dest={{ spark_folder }}/jars/jets3t-{{jets3t_version}}.jar timeout=1000 force=no
+
+- name: Download hadoop_aws and copy to Spark jars folder
+ become: yes
+ get_url: url={{ hadoop_aws_url }} dest={{ spark_folder }}/jars/hadoop-aws-{{hadoop_aws_version}}.jar timeout=1000 force=no
+
+- name: Download java_xmlbuilder and copy to Spark jars folder
+ become: yes
+ get_url: url={{ java_xmlbuilder_url }} dest={{ spark_folder }}/jars/java-xmlbuilder-{{java_xmlbuilder_version}}.jar timeout=1000 force=no
+
+- name: Download spark_cassandra_connector and copy to Spark jars folder
+ become: yes
+ get_url: url={{ spark_cassandra_connector_assembly_url }} dest={{ spark_folder }}/jars/spark-cassandra-connector-assembly_2.12-{{cassandra_connector_version}}.jar timeout=1000 force=no
+
+- name: Download common_pool_url and copy to Spark jars folder
+ become: yes
+ get_url: url={{ common_pool_url }} dest={{ spark_folder }}/jars/commons-pool2-{{commons_pool_version}}.jar timeout=1000 force=no
+
+
+- name: Download config to livy
+ command: hdfs dfs -get -f oci://{{ bucket }}@{{ oci_namespace }}/models-{{ model_version }}/application.conf {{ spark_folder }}/conf/application.conf
+
+- name: Download jets3t config to livy
+ command: hdfs dfs -get -f oci://{{ bucket }}@{{ oci_namespace }}/models-{{ model_version }}/jets3t.properties {{ spark_folder }}/conf/jets3t.properties
+
+
+- name: Update log4j.properties
+ become: yes
+ blockinfile:
+ path: "{{ spark_folder }}/conf/log4j.properties"
+ block: |
+ log4j.logger.org.ekstep.analytics=INFO
+ log4j.appender.org.ekstep.analytics=org.apache.log4j.RollingFileAppender
+ log4j.appender.org.ekstep.analytics.File=./joblog.log
+ log4j.appender.org.ekstep.analytics.MaxFileSize=${log4jspark.log.maxfilesize}
+ log4j.appender.org.ekstep.analytics.MaxBackupIndex=${log4jspark.log.maxbackupindex}
+ log4j.appender.org.ekstep.analytics.layout=org.apache.log4j.PatternLayout
+ log4j.appender.org.ekstep.analytics.layout.ConversionPattern=%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n
diff --git a/ansible/roles/redis-backup/tasks/main.yml b/ansible/roles/redis-backup/tasks/main.yml
index 061a3414ed..aafe94913c 100644
--- a/ansible/roles/redis-backup/tasks/main.yml
+++ b/ansible/roles/redis-backup/tasks/main.yml
@@ -49,6 +49,8 @@
name: gcp-cloud-storage
tasks_from: upload.yml
vars:
+ gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+ gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
gcp_bucket_name: "{{ cloud_storage_dpredisbackup_bucketname }}"
gcp_path: "{{ cloud_storage_dpredisbackup_foldername }}/{{ redis_backup_file_name }}"
local_file_or_folder_path: "{{ redis_backup_file_path }}"
diff --git a/ansible/roles/redis-multiprocess-backup/tasks/main.yml b/ansible/roles/redis-multiprocess-backup/tasks/main.yml
index 9e44b18f05..7e31c94adb 100644
--- a/ansible/roles/redis-multiprocess-backup/tasks/main.yml
+++ b/ansible/roles/redis-multiprocess-backup/tasks/main.yml
@@ -50,6 +50,8 @@
name: gcp-cloud-storage
tasks_from: upload-batch.yml
vars:
+ gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+ gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
gcp_bucket_name: "{{ cloud_storage_dpredisbackup_bucketname }}"
gcp_path: "{{ cloud_storage_dpredisbackup_foldername }}"
local_file_or_folder_path: "{{ redis_backup_dir }}/*"
diff --git a/ansible/roles/redis-multiprocess-restore/tasks/main.yml b/ansible/roles/redis-multiprocess-restore/tasks/main.yml
index 7483f4ac15..87f1838a19 100644
--- a/ansible/roles/redis-multiprocess-restore/tasks/main.yml
+++ b/ansible/roles/redis-multiprocess-restore/tasks/main.yml
@@ -38,6 +38,8 @@
name: gcp-cloud-storage
tasks_from: download.yml
vars:
+ gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+ gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
gcp_bucket_name: "{{ cloud_storage_dpredisbackup_bucketname }}"
gcp_path: "{{ cloud_storage_dpredisbackup_foldername }}/{{ item }}/{{ redis_restore_file_name }}"
local_file_or_folder_path: "/tmp/{{ item }}/{{ redis_restore_file_name }}"
diff --git a/ansible/roles/redis-restore/tasks/main.yml b/ansible/roles/redis-restore/tasks/main.yml
index bdab9bd116..a695f11e62 100644
--- a/ansible/roles/redis-restore/tasks/main.yml
+++ b/ansible/roles/redis-restore/tasks/main.yml
@@ -30,6 +30,8 @@
name: gcp-cloud-storage
tasks_from: download.yml
vars:
+ gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+ gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
gcp_bucket_name: "{{ cloud_storage_dpredisbackup_bucketname }}"
gcp_path: "{{ cloud_storage_dpredisbackup_foldername }}/{{ redis_restore_file_name }}"
dest_file_name: "{{ redis_restore_file_name }}"
diff --git a/ansible/roles/secor-telemetry-backup-deploy/defaults/main.yml b/ansible/roles/secor-telemetry-backup-deploy/defaults/main.yml
index a9edd50a3c..6f85c930d5 100644
--- a/ansible/roles/secor-telemetry-backup-deploy/defaults/main.yml
+++ b/ansible/roles/secor-telemetry-backup-deploy/defaults/main.yml
@@ -2,13 +2,14 @@ analytics_user: analytics
analytics_group: analytics
analytics_user_home: /home/{{analytics_user}}
sbin_path: "{{ analytics_user_home }}/sbin"
-azure_container_name: "{{secor_azure_container_name}}"
-azure_account_key: "{{sunbird_private_storage_account_key}}"
+azure_container_name: "{{cloud_storage_telemetry_bucketname}}"
+azure_account_key: "{{cloud_private_storage_secret}}"
telemetry_ingestion_topic: "{{ env }}.telemetry.ingest"
secor:
- properties: ['secor.azure', 'secor.common', 'secor', 'secor.partition', 'log4j']
+ properties:
+ ["secor.azure", "secor.common", "secor", "secor.partition", "log4j"]
artifact_dir: /mount/secor
secor_service_name:
@@ -81,7 +82,7 @@ secor_service_name:
max_file_age: "14400"
partition_prefix_enabled: "true"
partition_prefix_key: "eid"
- partition_prefix_mapping: "{\"ME_SESSION_SUMMARY\":\"summary\",\"ME_WORKFLOW_SUMMARY\":\"summary\",\"ME_DEIVCE_SUMMARY\":\"summary\",\"ME_ITEM_SUMMARY\":\"summary\",\"ME_WORKFLOW_USAGE_SUMMARY\":\"summary\",\"ME_ITEM_USAGE_SUMMARY\":\"summary\",\"ME_USAGE_SUMMARY\":\"summary\",\"ME_DIALCODE_USAGE_SUMMARY\":\"summary\",\"DEFAULT\":\"raw\"}"
+ partition_prefix_mapping: '{"ME_SESSION_SUMMARY":"summary","ME_WORKFLOW_SUMMARY":"summary","ME_DEIVCE_SUMMARY":"summary","ME_ITEM_SUMMARY":"summary","ME_WORKFLOW_USAGE_SUMMARY":"summary","ME_ITEM_USAGE_SUMMARY":"summary","ME_USAGE_SUMMARY":"summary","ME_DIALCODE_USAGE_SUMMARY":"summary","DEFAULT":"raw"}'
output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json"
service_description: "Denormalized Events Backup"
service_monitor_schedule_hr: "23"
@@ -106,7 +107,7 @@ secor_service_name:
service_monitor_schedule_hr: "23"
service_monitor_schedule_min: "50"
message_channel_identifier: ""
- message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+ message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
channel-telemetry-backup:
consumer_group: "{{ env }}.telemetry.channel.backup"
base_path: "data-exhaust/raw"
@@ -144,7 +145,7 @@ secor_service_name:
service_monitor_schedule_hr: "23"
service_monitor_schedule_min: "50"
message_channel_identifier: "derivedlocationdata.state"
- message_parser: "com.pinterest.secor.parser.ChannelDateMessageParser"
+ message_parser: "com.pinterest.secor.parser.ChannelDateMessageParser"
extractor-failed-backup:
consumer_group: "{{ env }}.extractor.failed.backup"
base_path: "extractor-failed"
@@ -252,7 +253,7 @@ secor_service_name:
max_file_age: "600"
partition_prefix_enabled: "true"
partition_prefix_key: "eid"
- partition_prefix_mapping: "{\"ME_SESSION_SUMMARY\":\"session_summary\",\"ME_WORKFLOW_SUMMARY\":\"workflow_summary\",\"ME_DEIVCE_SUMMARY\":\"device_summary\",\"ME_ITEM_SUMMARY\":\"item_summary\",\"ME_WORKFLOW_USAGE_SUMMARY\":\"workflow_usage_summary\",\"ME_ITEM_USAGE_SUMMARY\":\"item_usage_summary\",\"ME_USAGE_SUMMARY\":\"usage_summary\",\"ME_DIALCODE_USAGE_SUMMARY\":\"dialcode_usage_summary\",\"DEFAULT\":\"me\"}"
+ partition_prefix_mapping: '{"ME_SESSION_SUMMARY":"session_summary","ME_WORKFLOW_SUMMARY":"workflow_summary","ME_DEIVCE_SUMMARY":"device_summary","ME_ITEM_SUMMARY":"item_summary","ME_WORKFLOW_USAGE_SUMMARY":"workflow_usage_summary","ME_ITEM_USAGE_SUMMARY":"item_usage_summary","ME_USAGE_SUMMARY":"usage_summary","ME_DIALCODE_USAGE_SUMMARY":"dialcode_usage_summary","DEFAULT":"me"}'
output_file_pattern: "{partition}-{currentTimestamp}.json"
service_description: "Derived Telemetry Backup"
service_monitor_schedule_hr: "23"
@@ -312,7 +313,7 @@ secor_service_name:
max_file_age: "3600"
partition_prefix_enabled: "true"
partition_prefix_key: "jobName"
- partition_prefix_mapping: "{\"publish-pipeline\":\"publish_pipeline\",\"composite-search-indexer\":\"cs_index\",\"DEFAULT\":\"failed_events\"}"
+ partition_prefix_mapping: '{"publish-pipeline":"publish_pipeline","composite-search-indexer":"cs_index","DEFAULT":"failed_events"}'
output_file_pattern: "{partition}-{currentTimestamp}.json"
service_description: "Learning Failed Backup"
service_monitor_schedule_hr: "23"
@@ -413,4 +414,4 @@ secor_service_name:
service_monitor_schedule_hr: "23"
service_monitor_schedule_min: "57"
message_channel_identifier: ""
- message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
\ No newline at end of file
+ message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
diff --git a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.azure.j2 b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.azure.j2
index f44174563c..e55aa469c0 100644
--- a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.azure.j2
+++ b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.azure.j2
@@ -12,8 +12,8 @@ secor.azure.endpoints.protocol=https
# Microsoft Azure authentication credentials.
# https://azure.microsoft.com/en-us/documentation/articles/storage-create-storage-account
-secor.azure.account.name={{sunbird_private_storage_account_name}}
-secor.azure.account.key={{sunbird_private_storage_account_key}}
+secor.azure.account.name={{cloud_private_storage_accountname}}
+secor.azure.account.key={{cloud_private_storage_secret}}
# Microsoft Azure blob storage container name. Container is a grouping of a set
# of blobs. https://msdn.microsoft.com/en-us/library/dd135715.aspx
diff --git a/ansible/roles/zookeeper-upgrade/defaults/main.yml b/ansible/roles/zookeeper-upgrade/defaults/main.yml
index 6425d3340f..b8d6d3fd99 100644
--- a/ansible/roles/zookeeper-upgrade/defaults/main.yml
+++ b/ansible/roles/zookeeper-upgrade/defaults/main.yml
@@ -1,6 +1,6 @@
---
zookeeper_version: 3.6.3
-zookeeper_url: "https://downloads.apache.org/zookeeper/stable/apache-zookeeper-{{ zookeeper_version }}-bin.tar.gz"
+zookeeper_url: "https://archive.apache.org/dist/zookeeper/zookeeper-{{ zookeeper_version }}/apache-zookeeper-{{ zookeeper_version }}-bin.tar.gz"
zookeeper_port: 2181
zk_jvm_opts: "-Xms128m -Xmx128m"
verify: True
diff --git a/ansible/spark-cluster-job-submit.yml b/ansible/spark-cluster-job-submit.yml
index ba4e017a23..f21aa57860 100644
--- a/ansible/spark-cluster-job-submit.yml
+++ b/ansible/spark-cluster-job-submit.yml
@@ -4,8 +4,7 @@
vars_files:
- "{{inventory_dir}}/secrets.yml"
environment:
- AZURE_STORAGE_ACCOUNT: "{{sunbird_private_storage_account_name}}"
- AZURE_STORAGE_KEY: "{{sunbird_private_storage_account_key}}"
+ AZURE_STORAGE_ACCOUNT: "{{cloud_private_storage_accountname}}"
+ AZURE_STORAGE_KEY: "{{cloud_private_storage_secret}}"
roles:
- data-products-deploy
-
diff --git a/data-pipeline-flink/assessment-aggregator/pom.xml b/data-pipeline-flink/assessment-aggregator/pom.xml
index 2c4ca6f665..6b7a921a70 100644
--- a/data-pipeline-flink/assessment-aggregator/pom.xml
+++ b/data-pipeline-flink/assessment-aggregator/pom.xml
@@ -56,6 +56,12 @@
test
tests
+
+ com.squareup.okhttp3
+ mockwebserver
+ 4.4.0
+ test
+
it.ozimov
embedded-redis
diff --git a/data-pipeline-flink/assessment-aggregator/src/test/resources/forcevalidate.conf b/data-pipeline-flink/assessment-aggregator/src/test/resources/forcevalidate.conf
index 290d24809a..5f8ac8138f 100644
--- a/data-pipeline-flink/assessment-aggregator/src/test/resources/forcevalidate.conf
+++ b/data-pipeline-flink/assessment-aggregator/src/test/resources/forcevalidate.conf
@@ -36,5 +36,5 @@ redis {
}
assessment.skip.missingRecords = true
-content.read.api = "http://dev.sunbirded.org/api/content/v1/read/"
+content.read.api = "http://127.0.0.1:3000/api/content/v1/read/"
user.activity.agg.type="attempt_metrics"
\ No newline at end of file
diff --git a/data-pipeline-flink/assessment-aggregator/src/test/resources/test.conf b/data-pipeline-flink/assessment-aggregator/src/test/resources/test.conf
index 8e1b8bb5d1..fbe8e16690 100644
--- a/data-pipeline-flink/assessment-aggregator/src/test/resources/test.conf
+++ b/data-pipeline-flink/assessment-aggregator/src/test/resources/test.conf
@@ -35,5 +35,5 @@ redis {
}
}
assessment.skip.missingRecords = false
-content.read.api = "http://dev.sunbirded.org/api/content/v1/read/"
+content.read.api = "http://127.0.0.1:3000/api/content/v1/read/"
user.activity.agg.type="attempt_metrics"
\ No newline at end of file
diff --git a/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
index a5e048e865..40bba34aee 100644
--- a/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
+++ b/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
@@ -1,3 +1,4 @@
+
package org.sunbird.dp.fixture
object EventFixture {
@@ -24,7 +25,7 @@ object EventFixture {
//Recompute aggregate event
val RECOMPUTE_ASSESS_EVENT = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_2128415652377067521125","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","contentId":"do_212686723743318016173"}"""
-
+
val courseLeafNodes_1 = Map("do_2128415652377067521125:do_2128415652377067521125:leafnodes" -> "do_212686723743318016173")
val courseLeafNodes_2 = Map("do_312712196780204032110117:do_312712196780204032110117:leafnodes" -> "505c7c48ac6dc1edc9b08f21db5a571d")
val courseLeafNodes_3 = Map("do_2128410273679114241112:do_2128410273679114241112:leafnodes" -> "do_2128373396098744321673")
@@ -37,7 +38,7 @@ object EventFixture {
val DUPLICATE_BATCH_ASSESS_EVENTS_2 = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_873264782364827482","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_11307972307046400011917","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776_1","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776_2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776_3","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1626595233000,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1629273633000,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}"""
val DUPLICATE_BATCH_ASSESS_EVENTS_3 = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_873264782364827482","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_313026415363981312122123","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1626595233000,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1629273633000,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}"""
val DUPLICATE_BATCH_ASSESS_EVENTS_4 = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_87326478236482748244","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_1131998128479272961991","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1626595233000,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1629273633000,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}"""
- val contentCacheData_1 = Map("do_313026415363981312122" -> "{\n \"ownershipType\": [\n \"createdBy\"\n ],\n \"copyright\": \"EKSTEP\",\n \"previewUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/ecml/do_313026415363981312122-latest\",\n \"plugins\": [\n {\n \"identifier\": \"org.ekstep.stage\",\n \"semanticVersion\": \"1.0\"\n },\n {\n \"identifier\": \"org.ekstep.questionset\",\n \"semanticVersion\": \"1.0\"\n },\n {\n \"identifier\": \"org.ekstep.navigation\",\n \"semanticVersion\": \"1.0\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit\",\n \"semanticVersion\": \"1.1\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit.reorder\",\n \"semanticVersion\": \"1.1\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit.sequence\",\n \"semanticVersion\": \"1.1\"\n },\n {\n \"identifier\": \"org.ekstep.questionset.quiz\",\n \"semanticVersion\": \"1.0\"\n },\n {\n \"identifier\": \"org.ekstep.iterator\",\n \"semanticVersion\": \"1.0\"\n },\n {\n \"identifier\": \"org.ekstep.keyboard\",\n \"semanticVersion\": \"1.1\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit.ftb\",\n \"semanticVersion\": \"1.1\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit.mtf\",\n \"semanticVersion\": \"1.2\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit.mcq\",\n \"semanticVersion\": \"1.3\"\n },\n {\n \"identifier\": \"org.ekstep.summary\",\n \"semanticVersion\": \"1.0\"\n }\n ],\n \"subject\": [\n \"CBSE Training\"\n ],\n \"channel\": \"0123221758376673287017\",\n \"downloadUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_313026415363981312122/vjjjj_1590142788385_do_313026415363981312122_1.0.ecar\",\n \"questions\": [\n {\n \"name\": \"Arrange the Given sentence \\n\",\n \"relation\": \"associatedTo\",\n \"identifier\": \"do_313026448412631040111\",\n \"description\": null,\n \"objectType\": \"AssessmentItem\",\n \"status\": \"Live\"\n },\n {\n \"name\": \"Test data mark is 10\\n\",\n \"relation\": \"associatedTo\",\n \"identifier\": \"do_313026416457089024115\",\n \"description\": null,\n \"objectType\": \"AssessmentItem\",\n \"status\": \"Live\"\n },\n {\n \"name\": \"Arrange the given words in proper sentence\\n\",\n \"relation\": \"associatedTo\",\n \"identifier\": \"do_313026446885158912120\",\n \"description\": null,\n \"objectType\": \"AssessmentItem\",\n \"status\": \"Live\"\n },\n {\n \"name\": \"'The tree ____ is ____ and ____ mark is 3\\n\",\n \"relation\": \"associatedTo\",\n \"identifier\": \"do_313026423737737216117\",\n \"description\": null,\n \"objectType\": \"AssessmentItem\",\n \"status\": \"Live\"\n },\n {\n \"name\": \"Match the following\\n\",\n \"relation\": \"associatedTo\",\n \"identifier\": \"do_31302641810772787218\",\n \"description\": null,\n \"objectType\": \"AssessmentItem\",\n \"status\": \"Live\"\n }\n ],\n \"organisation\": [\n \"EKSTEP\"\n ],\n \"language\": [\n \"English\"\n ],\n \"mimeType\": \"application/vnd.ekstep.ecml-archive\",\n \"variants\": {\n \"spine\": {\n \"ecarUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_313026415363981312122/vjjjj_1590142788484_do_313026415363981312122_1.0_spine.ecar\",\n \"size\": 44094\n }\n },\n \"editorState\": \"{\\\"plugin\\\":{\\\"noOfExtPlugins\\\":11,\\\"extPlugins\\\":[{\\\"plugin\\\":\\\"org.ekstep.contenteditorfunctions\\\",\\\"version\\\":\\\"1.2\\\"},{\\\"plugin\\\":\\\"org.ekstep.keyboardshortcuts\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.richtext\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.iterator\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.navigation\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.reviewercomments\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.questionunit.mtf\\\",\\\"version\\\":\\\"1.2\\\"},{\\\"plugin\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.3\\\"},{\\\"plugin\\\":\\\"org.ekstep.keyboard\\\",\\\"version\\\":\\\"1.1\\\"},{\\\"plugin\\\":\\\"org.ekstep.questionunit.reorder\\\",\\\"version\\\":\\\"1.1\\\"},{\\\"plugin\\\":\\\"org.ekstep.questionunit.sequence\\\",\\\"version\\\":\\\"1.1\\\"}]},\\\"stage\\\":{\\\"noOfStages\\\":1,\\\"currentStage\\\":\\\"371a0b37-e8ab-4e8e-b83e-3775a2ed927d\\\",\\\"selectedPluginObject\\\":\\\"49469445-2ca2-479d-95b7-360aa07ee3bd\\\"},\\\"sidebar\\\":{\\\"selectedMenu\\\":\\\"settings\\\"}}\",\n \"objectType\": \"Content\",\n \"gradeLevel\": [\n \"Class 1\"\n ],\n \"appIcon\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_313026415363981312122/artifact/10_1560927487910.thumb.jpg\",\n \"primaryCategory\": \"Course Assessment\",\n \"collections\": [\n {\n \"name\": \"Copy of SelfServiceable\",\n \"relation\": \"hasSequenceMember\",\n \"identifier\": \"do_313026450799894528150\",\n \"description\": \"Execution\",\n \"objectType\": \"Collection\",\n \"status\": \"Retired\"\n }\n ],\n \"appId\": \"prod.diksha.portal\",\n \"contentEncoding\": \"gzip\",\n \"artifactUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_313026415363981312122/artifact/1590142788151_do_313026415363981312122.zip\",\n \"lockKey\": \"0708a132-4774-4126-b2e5-df0e83f6dd40\",\n \"sYS_INTERNAL_LAST_UPDATED_ON\": \"2020-05-22T10:19:50.208+0000\",\n \"contentType\": \"SelfAssess\",\n \"lastUpdatedBy\": \"2dded955-5656-4dc2-a683-e7e42082aa8c\",\n \"identifier\": \"do_313026415363981312122\",\n \"audience\": [\n \"Teacher\"\n ],\n \"visibility\": \"Default\",\n \"consumerId\": \"e85bcfb5-a8c2-4e65-87a2-0ebb43b45f01\",\n \"mediaType\": \"content\",\n \"osId\": \"org.ekstep.quiz.app\",\n \"lastPublishedBy\": \"5c627fa2-a7a4-490f-b9c8-bc0f42559562\",\n \"languageCode\": [\n \"en\"\n ],\n \"version\": 2,\n \"license\": \"CC BY 4.0\",\n \"prevState\": \"Review\",\n \"size\": 547518,\n \"lastPublishedOn\": \"2020-05-22T10:19:48.378+0000\",\n \"name\": \"Vjjjj\",\n \"status\": \"Retired\",\n \"totalQuestions\": 2,\n \"code\": \"org.sunbird.lfSsbq\",\n \"prevStatus\": \"Processing\",\n \"description\": \"Enter description for Assessment\",\n \"streamingUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/ecml/do_313026415363981312122-latest\",\n \"medium\": [\n \"English\"\n ],\n \"posterImage\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_3127871179775098881294/artifact/10_1560927487910.jpg\",\n \"idealScreenSize\": \"normal\",\n \"createdOn\": \"2020-05-22T09:09:54.237+0000\",\n \"copyrightYear\": 2020,\n \"contentDisposition\": \"inline\",\n \"licenseterms\": \"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content.\",\n \"lastUpdatedOn\": \"2020-05-22T11:59:41.479+0000\",\n \"dialcodeRequired\": \"No\",\n \"lastStatusChangedOn\": \"2020-05-22T11:59:41.479+0000\",\n \"createdFor\": [\n \"0123221758376673287017\"\n ],\n \"creator\": \"content_creator sahu\",\n \"os\": [\n \"All\"\n ],\n \"totalScore\": 5,\n \"pkgVersion\": 1,\n \"versionKey\": \"1590142785805\",\n \"idealScreenDensity\": \"hdpi\",\n \"framework\": \"ekstep_ncert_k-12\",\n \"s3Key\": \"ecar_files/do_313026415363981312122/vjjjj_1590142788385_do_313026415363981312122_1.0.ecar\",\n \"lastSubmittedOn\": \"2020-05-22T10:19:07.606+0000\",\n \"createdBy\": \"2dded955-5656-4dc2-a683-e7e42082aa8c\",\n \"compatibilityLevel\": 2,\n \"board\": \"CBSE\"\n }")
+ val contentCacheData_1 = Map("do_313026415363981312122" -> "{\n \"ownershipType\": [\n \"createdBy\"\n ],\n \"copyright\": \"EKSTEP\",\n \"previewUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/ecml/do_313026415363981312122-latest\",\n \"plugins\": [\n {\n \"identifier\": \"org.ekstep.stage\",\n \"semanticVersion\": \"1.0\"\n },\n {\n \"identifier\": \"org.ekstep.questionset\",\n \"semanticVersion\": \"1.0\"\n },\n {\n \"identifier\": \"org.ekstep.navigation\",\n \"semanticVersion\": \"1.0\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit\",\n \"semanticVersion\": \"1.1\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit.reorder\",\n \"semanticVersion\": \"1.1\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit.sequence\",\n \"semanticVersion\": \"1.1\"\n },\n {\n \"identifier\": \"org.ekstep.questionset.quiz\",\n \"semanticVersion\": \"1.0\"\n },\n {\n \"identifier\": \"org.ekstep.iterator\",\n \"semanticVersion\": \"1.0\"\n },\n {\n \"identifier\": \"org.ekstep.keyboard\",\n \"semanticVersion\": \"1.1\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit.ftb\",\n \"semanticVersion\": \"1.1\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit.mtf\",\n \"semanticVersion\": \"1.2\"\n },\n {\n \"identifier\": \"org.ekstep.questionunit.mcq\",\n \"semanticVersion\": \"1.3\"\n },\n {\n \"identifier\": \"org.ekstep.summary\",\n \"semanticVersion\": \"1.0\"\n }\n ],\n \"subject\": [\n \"CBSE Training\"\n ],\n \"channel\": \"0123221758376673287017\",\n \"downloadUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_313026415363981312122/vjjjj_1590142788385_do_313026415363981312122_1.0.ecar\",\n \"questions\": [\n {\n \"name\": \"Arrange the Given sentence \\n\",\n \"relation\": \"associatedTo\",\n \"identifier\": \"do_313026448412631040111\",\n \"description\": null,\n \"objectType\": \"AssessmentItem\",\n \"status\": \"Live\"\n },\n {\n \"name\": \"Test data mark is 10\\n\",\n \"relation\": \"associatedTo\",\n \"identifier\": \"do_313026416457089024115\",\n \"description\": null,\n \"objectType\": \"AssessmentItem\",\n \"status\": \"Live\"\n },\n {\n \"name\": \"Arrange the given words in proper sentence\\n\",\n \"relation\": \"associatedTo\",\n \"identifier\": \"do_313026446885158912120\",\n \"description\": null,\n \"objectType\": \"AssessmentItem\",\n \"status\": \"Live\"\n },\n {\n \"name\": \"'The tree ____ is ____ and ____ mark is 3\\n\",\n \"relation\": \"associatedTo\",\n \"identifier\": \"do_313026423737737216117\",\n \"description\": null,\n \"objectType\": \"AssessmentItem\",\n \"status\": \"Live\"\n },\n {\n \"name\": \"Match the following\\n\",\n \"relation\": \"associatedTo\",\n \"identifier\": \"do_31302641810772787218\",\n \"description\": null,\n \"objectType\": \"AssessmentItem\",\n \"status\": \"Live\"\n }\n ],\n \"organisation\": [\n \"EKSTEP\"\n ],\n \"language\": [\n \"English\"\n ],\n \"mimeType\": \"application/vnd.ekstep.ecml-archive\",\n \"variants\": {\n \"spine\": {\n \"ecarUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_313026415363981312122/vjjjj_1590142788484_do_313026415363981312122_1.0_spine.ecar\",\n \"size\": 44094\n }\n },\n \"editorState\": \"{\\\"plugin\\\":{\\\"noOfExtPlugins\\\":11,\\\"extPlugins\\\":[{\\\"plugin\\\":\\\"org.ekstep.contenteditorfunctions\\\",\\\"version\\\":\\\"1.2\\\"},{\\\"plugin\\\":\\\"org.ekstep.keyboardshortcuts\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.richtext\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.iterator\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.navigation\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.reviewercomments\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.questionunit.mtf\\\",\\\"version\\\":\\\"1.2\\\"},{\\\"plugin\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.3\\\"},{\\\"plugin\\\":\\\"org.ekstep.keyboard\\\",\\\"version\\\":\\\"1.1\\\"},{\\\"plugin\\\":\\\"org.ekstep.questionunit.reorder\\\",\\\"version\\\":\\\"1.1\\\"},{\\\"plugin\\\":\\\"org.ekstep.questionunit.sequence\\\",\\\"version\\\":\\\"1.1\\\"}]},\\\"stage\\\":{\\\"noOfStages\\\":1,\\\"currentStage\\\":\\\"371a0b37-e8ab-4e8e-b83e-3775a2ed927d\\\",\\\"selectedPluginObject\\\":\\\"49469445-2ca2-479d-95b7-360aa07ee3bd\\\"},\\\"sidebar\\\":{\\\"selectedMenu\\\":\\\"settings\\\"}}\",\n \"objectType\": \"Content\",\n \"gradeLevel\": [\n \"Class 1\"\n ],\n \"appIcon\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_313026415363981312122/artifact/10_1560927487910.thumb.jpg\",\n \"primaryCategory\": \"Course Assessment\",\n \"collections\": [\n {\n \"name\": \"Copy of SelfServiceable\",\n \"relation\": \"hasSequenceMember\",\n \"identifier\": \"do_313026450799894528150\",\n \"description\": \"Execution\",\n \"objectType\": \"Collection\",\n \"status\": \"Retired\"\n }\n ],\n \"appId\": \"prod.diksha.portal\",\n \"contentEncoding\": \"gzip\",\n \"artifactUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_313026415363981312122/artifact/1590142788151_do_313026415363981312122.zip\",\n \"lockKey\": \"0708a132-4774-4126-b2e5-df0e83f6dd40\",\n \"sYS_INTERNAL_LAST_UPDATED_ON\": \"2020-05-22T10:19:50.208+0000\",\n \"contentType\": \"SelfAssess\",\n \"lastUpdatedBy\": \"2dded955-5656-4dc2-a683-e7e42082aa8c\",\n \"identifier\": \"do_313026415363981312122\",\n \"audience\": [\n \"Teacher\"\n ],\n \"visibility\": \"Default\",\n \"consumerId\": \"e85bcfb5-a8c2-4e65-87a2-0ebb43b45f01\",\n \"mediaType\": \"content\",\n \"osId\": \"org.ekstep.quiz.app\",\n \"lastPublishedBy\": \"5c627fa2-a7a4-490f-b9c8-bc0f42559562\",\n \"languageCode\": [\n \"en\"\n ],\n \"version\": 2,\n \"license\": \"CC BY 4.0\",\n \"prevState\": \"Review\",\n \"size\": 547518,\n \"lastPublishedOn\": \"2020-05-22T10:19:48.378+0000\",\n \"name\": \"Vjjjj\",\n \"status\": \"Retired\",\n \"totalQuestions\": 2,\n \"code\": \"org.sunbird.lfSsbq\",\n \"prevStatus\": \"Processing\",\n \"description\": \"Enter description for Assessment\",\n \"streamingUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/ecml/do_313026415363981312122-latest\",\n \"medium\": [\n \"English\"\n ],\n \"posterImage\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_3127871179775098881294/artifact/10_1560927487910.jpg\",\n \"idealScreenSize\": \"normal\",\n \"createdOn\": \"2020-05-22T09:09:54.237+0000\",\n \"copyrightYear\": 2020,\n \"contentDisposition\": \"inline\",\n \"licenseterms\": \"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content.\",\n \"lastUpdatedOn\": \"2020-05-22T11:59:41.479+0000\",\n \"dialcodeRequired\": \"No\",\n \"lastStatusChangedOn\": \"2020-05-22T11:59:41.479+0000\",\n \"createdFor\": [\n \"0123221758376673287017\"\n ],\n \"creator\": \"content_creator sahu\",\n \"os\": [\n \"All\"\n ],\n \"totalScore\": 5,\n \"pkgVersion\": 1,\n \"versionKey\": \"1590142785805\",\n \"idealScreenDensity\": \"hdpi\",\n \"framework\": \"ekstep_ncert_k-12\",\n \"s3Key\": \"ecar_files/do_313026415363981312122/vjjjj_1590142788385_do_313026415363981312122_1.0.ecar\",\n \"lastSubmittedOn\": \"2020-05-22T10:19:07.606+0000\",\n \"createdBy\": \"2dded955-5656-4dc2-a683-e7e42082aa8c\",\n \"compatibilityLevel\": 2,\n \"board\": \"CBSE\"\n }")
val contentCacheData_2 = Map("do_313026415363981312122123" -> "{\"totalQuestions\":1}")
val contentCacheList = List(contentCacheData_1, contentCacheData_2)
-}
+}
\ No newline at end of file
diff --git a/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/spec/AssessmentAggregatorTaskTestSpec.scala b/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/spec/AssessmentAggregatorTaskTestSpec.scala
index 171f7e4f71..3cea7b170b 100644
--- a/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/spec/AssessmentAggregatorTaskTestSpec.scala
+++ b/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/spec/AssessmentAggregatorTaskTestSpec.scala
@@ -27,6 +27,8 @@ import org.sunbird.dp.core.util.{CassandraUtil, JSONUtil}
import org.sunbird.dp.fixture.EventFixture
import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec}
import redis.embedded.RedisServer
+import okhttp3.mockwebserver.{MockResponse, MockWebServer}
+import java.io.IOException
class AssessmentAggregatorTaskTestSpec extends BaseTestSpec {
@@ -44,6 +46,7 @@ class AssessmentAggregatorTaskTestSpec extends BaseTestSpec {
val assessmentConfig: AssessmentAggregatorConfig = new AssessmentAggregatorConfig(config)
val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable())
val gson = new Gson()
+ val server = new MockWebServer()
var cassandraUtil: CassandraUtil = _
@@ -56,6 +59,7 @@ class AssessmentAggregatorTaskTestSpec extends BaseTestSpec {
EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L)
cassandraUtil = new CassandraUtil(assessmentConfig.dbHost, assessmentConfig.dbPort)
val session = cassandraUtil.session
+ setupRestUtilData()
setupRedisTestData()
val dataLoader = new CQLDataLoader(session)
@@ -72,6 +76,7 @@ class AssessmentAggregatorTaskTestSpec extends BaseTestSpec {
redisServer.stop()
try {
EmbeddedCassandraServerHelper.cleanEmbeddedCassandra()
+ server.close()
} catch {
case ex: Exception => {
@@ -80,6 +85,21 @@ class AssessmentAggregatorTaskTestSpec extends BaseTestSpec {
flinkCluster.after()
}
+ def setupRestUtilData(): Unit = {
+ val do_11307972307046400011917_response = """{"id":"api.content.read","ver":"1.0","ts":"2023-05-17T10:26:51.549Z","params":{"resmsgid":"566eacd0-f49d-11ed-bf1e-7fae1bdbcdf8","msgid":"566de980-f49d-11ed-8721-d532b5857c8a","status":"successful","err":null,"errmsg":null},"responseCode":"OK","result":{"content":{"ownershipType":["createdBy"],"copyright":"Sunbird Org","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_113762457691021312168/samplevideo_1280x720_1mb.mp4","channel":"0137541424673095687","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_113762457691021312168/content-2_1679987660391_do_113762457691021312168_1.ecar","organisation":["Sunbird Org"],"language":["English"],"mimeType":"video/mp4","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_113762457691021312168/content-2_1679987660391_do_113762457691021312168_1.ecar","size":"1058720"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_113762457691021312168/content-2_1679987660892_do_113762457691021312168_1_SPINE.ecar","size":"4153"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_113762457691021312168/artifact/do_11376182453272576019_1679910221428_287-2876925_test-image-png-unit-testing-png-transparent-png.thumb.png","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_113762457691021312168/samplevideo_1280x720_1mb.mp4","lockKey":"1a558750-41dd-43d3-9d4f-9f12184a902e","contentType":"Resource","category2":"Category2 Term1","identifier":"do_113762457691021312168","lastUpdatedBy":"155ce3c5-713e-4749-bc1c-95d09c640914","category3":"Category3 Term1","audience":["Student"],"category4":"Category4 Term1","category5":"Category5 Term1","visibility":"Default","category1":"Category1 Term1","discussionForum":{"enabled":"No"},"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"469dc732-04f3-42d9-9a85-30957a797acc","version":2,"license":"CC BY 4.0","prevState":"Review","size":1055736,"lastPublishedOn":"2023-03-28T07:14:20.009+0000","name":"Content - 2","status":"Live","code":"62ada120-13c4-4e94-aad6-56cebe6a089c","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirdspikemedia-inct.streaming.media.azure.net/5d2643e3-fcae-42a8-8a22-ac291a317ed4/samplevideo_1280x720_1mb.ism/manifest(format=m3u8-aapl-v3)","posterImage":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11376182453272576019/artifact/do_11376182453272576019_1679910221428_287-2876925_test-image-png-unit-testing-png-transparent-png.png","idealScreenSize":"normal","createdOn":"2023-03-28T07:11:51.115+0000","contentDisposition":"inline","lastUpdatedOn":"2023-03-28T07:44:16.190+0000","dialcodeRequired":"No","lastStatusChangedOn":"2023-03-28T07:14:21.098+0000","createdFor":["0137541424673095687"],"creator":"contentCreator Creator","os":["All"],"se_FWIds":["NCF"],"pkgVersion":1,"versionKey":"1679989456190","idealScreenDensity":"hdpi","framework":"framework1","lastSubmittedOn":"2023-03-28T07:12:10.623+0000","createdBy":"155ce3c5-713e-4749-bc1c-95d09c640914","compatibilityLevel":1,"resourceType":"Learn"}}}"""
+ val do_1131998128479272961991_response = """{"id":"api.content.read","ver":"1.0","ts":"2023-05-17T11:08:52.532Z","params":{"resmsgid":"350e2740-f4a3-11ed-bf1e-7fae1bdbcdf8","msgid":"350d15d0-f4a3-11ed-8721-d532b5857c8a","status":"successful","err":null,"errmsg":null},"responseCode":"OK","result":{"content":{"ownershipType":["createdBy"],"copyright":"sunbird","se_gradeLevelIds":["ncf_gradelevel_class1"],"subject":["Telugu"],"channel":"0137541424673095687","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11376180991508480011/test-book_1679910340314_do_11376180991508480011_1_SPINE.ecar","organisation":["Sunbird Org"],"language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11376180991508480011/test-book_1679910340314_do_11376180991508480011_1_SPINE.ecar","size":"8922"},"online":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11376180991508480011/test-book_1679910340572_do_11376180991508480011_1_ONLINE.ecar","size":"5179"}},"leafNodes":["do_11376182093890355216","do_11376182438513868818"],"objectType":"Content","se_mediums":["Telugu"],"gradeLevel":["Class 1"],"appIcon":"","primaryCategory":"Digital Textbook","contentEncoding":"gzip","lockKey":"fbeda787-5742-4c45-a535-fbb2c99ac3a0","generateDIALCodes":"Yes","totalCompressedSize":3579342,"mimeTypesCount":"{\"video/mp4\":2,\"application/vnd.ekstep.content-collection\":2}","sYS_INTERNAL_LAST_UPDATED_ON":"2023-03-27T09:45:40.314+0000","contentType":"TextBook","se_gradeLevels":["Class 1"],"trackable":{"enabled":"No","autoBatch":"No"},"identifier":"do_11376180991508480011","audience":["Student"],"se_boardIds":["ncf_board_other"],"subjectIds":["ncf_subject_telugu"],"toc_url":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11376180991508480011/artifact/do_11376180991508480011_toc.json","visibility":"Default","contentTypesCount":"{\"TextBookUnit\":2,\"Resource\":2}","author":"BookCreator bookCreator","consumerId":"bfe5883f-ac66-4744-a064-3ed88d986eba","childNodes":["do_11376182093890355216","do_11376181820567552012","do_11376182438513868818","do_11376181820809216014"],"discussionForum":{"enabled":"No"},"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"4b4dda54-b061-4346-9aaa-e2801430b885","version":2,"se_subjects":["Telugu"],"license":"CC BY 4.0","prevState":"Review","size":8922,"lastPublishedOn":"2023-03-27T09:45:40.159+0000","name":"Test Book","mediumIds":["ncf_medium_telugu"],"status":"Live","code":"org.sunbird.jMNK3Z","credentials":{"enabled":"No"},"prevStatus":"Processing","description":"Enter description for TextBook","medium":["Telugu"],"idealScreenSize":"normal","createdOn":"2023-03-27T09:13:56.965+0000","se_boards":["Other"],"se_mediumIds":["ncf_medium_telugu"],"copyrightYear":2023,"contentDisposition":"inline","lastUpdatedOn":"2023-03-27T09:45:40.820+0000","dialcodeRequired":"No","lastStatusChangedOn":"2023-03-27T09:45:40.820+0000","createdFor":["0137541424673095687"],"creator":"BookCreator bookCreator","os":["All"],"se_subjectIds":["ncf_subject_telugu"],"se_FWIds":["NCF"],"pkgVersion":1,"versionKey":"1679910320288","idealScreenDensity":"hdpi","framework":"NCF","depth":0,"s3Key":"content/do_11376180991508480011/artifact/do_11376180991508480011_toc.json","boardIds":["ncf_board_other"],"lastSubmittedOn":"2023-03-27T09:45:20.280+0000","createdBy":"7bf81b8b-ab64-47ca-b9d7-c9f74f811980","compatibilityLevel":1,"leafNodesCount":2,"userConsent":"Yes","gradeLevelIds":["ncf_gradelevel_class1"],"board":"Other","resourceType":"Book"}}}"""
+ try {
+ server.start(3000)
+ } catch {
+ case e: IOException =>
+ System.out.println("Exception" + e)
+ }
+ server.enqueue(new MockResponse().setBody(do_11307972307046400011917_response))
+ server.url("http://127.0.0.1:3000/api/content/v1/read/do_11307972307046400011917")
+ server.enqueue(new MockResponse().setBody(do_1131998128479272961991_response))
+ server.url("http://127.0.0.1:3000/api/content/v1/read/do_1131998128479272961991")
+ }
+
"AssessmentAggregator " should "Update event to db" in {
when(mockKafkaUtil.kafkaEventSource[Event](assessmentConfig.kafkaInputTopic)).thenReturn(new AssessmentAggreagatorEventSource)
diff --git a/data-pipeline-flink/device-profile-updater/src/main/scala/org/sunbird/dp/deviceprofile/functions/DeviceProfileUpdaterFunction.scala b/data-pipeline-flink/device-profile-updater/src/main/scala/org/sunbird/dp/deviceprofile/functions/DeviceProfileUpdaterFunction.scala
index c5a9ac8fa7..9a3bcea499 100644
--- a/data-pipeline-flink/device-profile-updater/src/main/scala/org/sunbird/dp/deviceprofile/functions/DeviceProfileUpdaterFunction.scala
+++ b/data-pipeline-flink/device-profile-updater/src/main/scala/org/sunbird/dp/deviceprofile/functions/DeviceProfileUpdaterFunction.scala
@@ -51,7 +51,8 @@ class DeviceProfileUpdaterFunction(config: DeviceProfileUpdaterConfig,
database = config.postgresDb,
host = config.postgresHost,
port = config.postgresPort,
- maxConnections = config.postgresMaxConnections
+ maxConnections = config.postgresMaxConnections,
+ sslMode = config.postgresSslMode
))
}
}
diff --git a/data-pipeline-flink/device-profile-updater/src/main/scala/org/sunbird/dp/deviceprofile/task/DeviceProfileUpdaterConfig.scala b/data-pipeline-flink/device-profile-updater/src/main/scala/org/sunbird/dp/deviceprofile/task/DeviceProfileUpdaterConfig.scala
index 68c32fa873..df84b5aeb6 100644
--- a/data-pipeline-flink/device-profile-updater/src/main/scala/org/sunbird/dp/deviceprofile/task/DeviceProfileUpdaterConfig.scala
+++ b/data-pipeline-flink/device-profile-updater/src/main/scala/org/sunbird/dp/deviceprofile/task/DeviceProfileUpdaterConfig.scala
@@ -35,6 +35,7 @@ class DeviceProfileUpdaterConfig(override val config: Config) extends BaseJobCon
val postgresHost: String = config.getString("postgres.host")
val postgresPort: Int = config.getInt("postgres.port")
val postgresMaxConnections: Int = config.getInt("postgres.maxConnections")
+ val postgresSslMode: Boolean = config.getBoolean("postgres.sslMode")
val countryCode = "country_code"
diff --git a/data-pipeline-flink/dp-core/pom.xml b/data-pipeline-flink/dp-core/pom.xml
index cfe8164c5f..a33c0f93e4 100644
--- a/data-pipeline-flink/dp-core/pom.xml
+++ b/data-pipeline-flink/dp-core/pom.xml
@@ -64,11 +64,16 @@
cassandra-driver-core
3.7.0
-
+
+
+ org.postgresql
+ postgresql
+ 42.6.0
+
org.scalatest
scalatest_2.12
diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/PostgresConnect.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/PostgresConnect.scala
index 0705498767..3c8393a651 100644
--- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/PostgresConnect.scala
+++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/PostgresConnect.scala
@@ -3,8 +3,9 @@ package org.sunbird.dp.core.util
import java.sql.{Connection, ResultSet, SQLException}
import org.postgresql.ds.PGPoolingDataSource
+import org.postgresql.ssl.NonValidatingFactory
-final case class PostgresConnectionConfig(user: String, password: String, database: String, host: String, port: Int, maxConnections: Int)
+final case class PostgresConnectionConfig(user: String, password: String, database: String, host: String, port: Int, maxConnections: Int,sslMode: Boolean = false)
class PostgresConnect(config: PostgresConnectionConfig) {
@@ -26,6 +27,11 @@ class PostgresConnect(config: PostgresConnectionConfig) {
source.setPassword(config.password)
source.setDatabaseName(config.database)
source.setMaxConnections(config.maxConnections)
+ if (config.sslMode) {
+ source.setProperty("ssl", "true")
+ source.setProperty("sslmode", "require")
+ source.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory")
+ }
}
@throws[Exception]
@@ -42,7 +48,9 @@ class PostgresConnect(config: PostgresConnectionConfig) {
@throws[Exception]
def closeConnection(): Unit = {
connection.close()
- source.close()
+ if (Option(source).isEmpty) {
+ source.close()
+ }
}
@throws[Exception]
diff --git a/data-pipeline-flink/dp-core/src/test/resources/base-test.conf b/data-pipeline-flink/dp-core/src/test/resources/base-test.conf
index 2c678d1993..a1de03c33d 100644
--- a/data-pipeline-flink/dp-core/src/test/resources/base-test.conf
+++ b/data-pipeline-flink/dp-core/src/test/resources/base-test.conf
@@ -50,6 +50,8 @@ postgres {
maxConnections = 2
user = "postgres"
password = "postgres"
+ sslMode = false
+
}
lms-cassandra {
diff --git a/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/spec/PostgresConnectSpec.scala b/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/spec/PostgresConnectSpec.scala
index fdbfe5ddf3..5d22ffed7a 100644
--- a/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/spec/PostgresConnectSpec.scala
+++ b/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/spec/PostgresConnectSpec.scala
@@ -17,15 +17,14 @@ class PostgresConnectSpec extends BaseSpec with Matchers with MockitoSugar {
database = "postgres",
host = config.getString("postgres.host"),
port = config.getInt("postgres.port"),
- maxConnections = config.getInt("postgres.maxConnections")
+ maxConnections = config.getInt("postgres.maxConnections"),
+ sslMode = config.getBoolean("postgres.sslMode")
)
val postgresConnect = new PostgresConnect(postgresConfig)
val connection = postgresConnect.getConnection
connection should not be (null)
-
postgresConnect.execute("CREATE TABLE device_table(id text PRIMARY KEY, channel text);")
postgresConnect.execute("INSERT INTO device_table(id,channel) VALUES('12345','custchannel');")
-
val st = connection.createStatement
val rs = postgresConnect.executeQuery("SELECT * FROM device_table where id='12345';")
while ( {
@@ -46,7 +45,8 @@ class PostgresConnectSpec extends BaseSpec with Matchers with MockitoSugar {
database = "postgres",
host = config.getString("postgres.host"),
port = config.getInt("postgres.port"),
- maxConnections = config.getInt("postgres.maxConnections")
+ maxConnections = config.getInt("postgres.maxConnections"),
+ sslMode = config.getBoolean("postgres.sslMode")
)
val postgresConnect = new PostgresConnect(postgresConfig)
val connection = postgresConnect.getConnection
diff --git a/kubernetes/ansible/bootstrap_minimal.yaml b/kubernetes/ansible/bootstrap_minimal.yaml
index f9facd6152..ae42a2b35e 100644
--- a/kubernetes/ansible/bootstrap_minimal.yaml
+++ b/kubernetes/ansible/bootstrap_minimal.yaml
@@ -33,7 +33,7 @@
ignore_errors: yes
- name: Creating docker secrets
- shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ core_vault_docker_registry_url }} --docker-username {{ core_vault_docker_registry_user }} --docker-password {{ core_vault_docker_registry_password }} --dry-run=client -o=yaml | kubectl apply -f -"
+ shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ core_vault_docker_registry_url }} --docker-username {{ core_vault_docker_registry_user }} --docker-password '{{ core_vault_docker_registry_password }}' --dry-run=client -o=yaml | kubectl apply -f -"
with_items:
- "flink-{{ env }}"
- "flink-kp-{{ env }}"
diff --git a/kubernetes/ansible/roles/druid/defaults/main.yml b/kubernetes/ansible/roles/druid/defaults/main.yml
index 509b3d6dcf..98dd6c092f 100644
--- a/kubernetes/ansible/roles/druid/defaults/main.yml
+++ b/kubernetes/ansible/roles/druid/defaults/main.yml
@@ -128,9 +128,9 @@ druid_configs:
druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}"
#Druid Azure Details
druid_storage_type: "azure"
- azure_account_name: "{{ sunbird_druid_storage_account_name }}"
- azure_storage_secret: "{{ sunbird_druid_storage_account_key }}"
- azure_container: "{{ druid_azure_container_name }}"
+ azure_account_name: "{{ cloud_private_storage_accountname }}"
+ azure_storage_secret: "{{ cloud_private_storage_secret }}"
+ azure_container: "{{ cloud_storage_telemetry_bucketname }}"
#Logging the indexing logs to azure
druid_log_azure_container: "{{ druid_azure_container_name }}"
druid_log_azure_folder: "druidlogs"
@@ -206,9 +206,9 @@ druid_configs:
druid_postgres_user: "druid@{{ postgres.db_url }}"
#Druid Azure Details
druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}"
- azure_account_name: "{{ sunbird_druid_storage_account_name }}"
- azure_storage_secret: "{{ sunbird_druid_storage_account_key }}"
- azure_container: "{{ druid_azure_container_name }}"
+ azure_account_name: "{{ cloud_private_storage_accountname }}"
+ azure_storage_secret: "{{ cloud_private_storage_secret }}"
+ azure_container: "{{ cloud_storage_telemetry_bucketname }}"
#Logging the indexing logs to azure
druid_log_azure_container: "{{ druid_azure_container_name }}"
druid_log_azure_folder: "druidlogs"
@@ -266,4 +266,4 @@ druid_configs:
druid_router_http_numMaxThreads: 100
druid_server_http_numThreads: 100
druid_router_managementProxy_enabled: true
- druid_router_replicas: 1
\ No newline at end of file
+ druid_router_replicas: 1
diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml
index 633026b825..17bd0257bb 100644
--- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml
+++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml
@@ -1,6 +1,6 @@
flink_namespace: "flink-{{ env }}"
imagepullsecrets: "{{ env }}-registry-secret"
-service_monitor_enabled: true
+service_monitor_enabled: false
### Job manager related vars
jobmanager_rpc_port: 6123
jobmanager_blob_port: 6124
@@ -30,11 +30,11 @@ taskmanager_heap_memory: 1024
### base-config related vars
postgres_max_connections: 2
-azure_account: "{{ sunbird_private_storage_account_name }}"
-azure_secret: "{{ sunbird_private_storage_account_key }}"
-flink_container_name: dev-data-store
+azure_account: "{{ cloud_private_storage_accountname }}"
+azure_secret: "{{ cloud_private_storage_secret }}"
+cloud_storage_flink_bucketname: dev-data-store
flink_dp_storage_container: ""
-checkpoint_store_type: azure
+checkpoint_store_type: gcloud
checkpoint_interval: 60000
checkpoint_pause_between_seconds: 5000
checkpoint_compression_enabled: true
diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrole.yaml b/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrole.yaml
index 8d51ef406b..b2817c5f9a 100755
--- a/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrole.yaml
+++ b/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrole.yaml
@@ -1,5 +1,9 @@
{{- if and .Values.reloader.watchGlobally (.Values.reloader.rbac.enabled) }}
+{{- if (.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1") }}
+apiVersion: rbac.authorization.k8s.io/v1
+{{ else }}
apiVersion: rbac.authorization.k8s.io/v1beta1
+{{- end }}
kind: ClusterRole
metadata:
labels:
diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrolebinding.yaml b/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrolebinding.yaml
index 28c9d4b916..748e52528d 100755
--- a/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrolebinding.yaml
+++ b/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrolebinding.yaml
@@ -1,5 +1,9 @@
{{- if and .Values.reloader.watchGlobally (.Values.reloader.rbac.enabled) }}
+{{- if (.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1") }}
+apiVersion: rbac.authorization.k8s.io/v1
+{{ else }}
apiVersion: rbac.authorization.k8s.io/v1beta1
+{{- end }}
kind: ClusterRoleBinding
metadata:
labels:
diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/role.yaml b/kubernetes/helm_charts/bootstrap/reloader/templates/role.yaml
index 5827f5cdcb..cbd5979451 100755
--- a/kubernetes/helm_charts/bootstrap/reloader/templates/role.yaml
+++ b/kubernetes/helm_charts/bootstrap/reloader/templates/role.yaml
@@ -1,5 +1,5 @@
{{- if and (not (.Values.reloader.watchGlobally)) (.Values.reloader.rbac.enabled) }}
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/rolebinding.yaml b/kubernetes/helm_charts/bootstrap/reloader/templates/rolebinding.yaml
index 94fb1f838b..08868f0e06 100755
--- a/kubernetes/helm_charts/bootstrap/reloader/templates/rolebinding.yaml
+++ b/kubernetes/helm_charts/bootstrap/reloader/templates/rolebinding.yaml
@@ -1,5 +1,5 @@
{{- if and (not (.Values.reloader.watchGlobally)) (.Values.reloader.rbac.enabled) }}
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml
index 10e6b62181..35d9066161 100644
--- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml
+++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml
@@ -111,13 +111,18 @@ spec:
args: ["start-foreground",
"--job-classname={{ .Values.job_classname }}",
{{- if eq .Values.checkpoint_store_type "azure" }}
- "-Dfs.azure.account.key.{{ .Values.azure_account }}.blob.core.windows.net={{ .Values.azure_secret }}",
+ "-Dfs.azure.account.key.{{ .Values.cloud_storage_key }}.blob.core.windows.net={{ .Values.cloud_storage_secret }}",
{{- end }}
{{- if eq .Values.checkpoint_store_type "s3" }}
- "-Ds3.access-key={{ .Values.s3_access_key }}",
- "-Ds3.secret-key={{ .Values.s3_secret_key }}",
- "-Ds3.endpoint={{ .Values.s3_endpoint }}",
+ "-Ds3.access-key={{ .Values.cloud_storage_key }}",
+ "-Ds3.secret-key={{ .Values.cloud_storage_secret }}",
+ "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}",
"-Ds3.path.style.access={{ .Values.s3_path_style_access }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "gcloud" }}
+ "-Dfs.gs.auth.client.id={{ .Values.cloud_storage_key }}",
+ "-Dfs.gs.auth.client.secret={{ .Values.cloud_storage_secret }}",
+ "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}",
{{- end }}
"-Dweb.submit.enable=false",
"-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter",
@@ -192,13 +197,18 @@ spec:
command: ["/opt/flink/bin/taskmanager.sh"]
args: ["start-foreground",
{{- if eq .Values.checkpoint_store_type "azure" }}
- "-Dfs.azure.account.key.{{ .Values.azure_account }}.blob.core.windows.net={{ .Values.azure_secret }}",
+ "-Dfs.azure.account.key.{{ .Values.cloud_storage_key }}.blob.core.windows.net={{ .Values.cloud_storage_secret }}",
{{- end }}
{{- if eq .Values.checkpoint_store_type "s3" }}
- "-Ds3.access-key={{ .Values.s3_access_key }}",
- "-Ds3.secret-key={{ .Values.s3_secret_key }}",
- "-Ds3.endpoint={{ .Values.s3_endpoint }}",
+ "-Ds3.access-key={{ .Values.cloud_storage_key }}",
+ "-Ds3.secret-key={{ .Values.cloud_storage_secret }}",
+ "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}",
"-Ds3.path.style.access={{ .Values.s3_path_style_access }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "gcloud" }}
+ "-Dfs.gs.auth.client.id={{ .Values.cloud_storage_key }}",
+ "-Dfs.gs.auth.client.secret={{ .Values.cloud_storage_secret }}",
+ "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}",
{{- end }}
"-Dweb.submit.enable=false",
"-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter",
diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2
index 22630c3015..4e2920caf7 100644
--- a/kubernetes/helm_charts/datapipeline_jobs/values.j2
+++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2
@@ -8,9 +8,13 @@ azure_account: {{ azure_account }}
azure_secret: {{ azure_secret }}
s3_access_key: {{ s3_storage_key }}
s3_secret_key: {{ s3_storage_secret }}
+{% if cloud_service_provider == "oci" %}
+s3_endpoint: {{ oci_flink_s3_storage_endpoint }}
+{% else %}
s3_endpoint: {{ s3_storage_endpoint }}
+{% endif %}
s3_path_style_access: {{ s3_path_style_access }}
-
+cloud_storage_project_id: {{ cloud_public_storage_project }}
serviceMonitor:
enabled: {{ service_monitor_enabled | lower}}
replicaCount: {{taskmanager_replicacount|default(1)}}
@@ -150,8 +154,8 @@ base_config: |
statebackend {
blob {
storage {
- account = "{% if checkpoint_store_type == "azure" %}{{ azure_account }}.blob.core.windows.net{% elif checkpoint_store_type == "s3" %}{{ flink_dp_storage_container }}{% endif %}"
- container = "{{ flink_container_name }}"
+ account = "{% if checkpoint_store_type == "azure" %}{{ azure_account }}.blob.core.windows.net{% elif checkpoint_store_type == "s3" %}{{ flink_dp_storage_container }}{% elif checkpoint_store_type == "gcloud" %}{{ flink_dp_storage_container }}{% endif %}"
+ container = "{{ cloud_storage_flink_bucketname }}"
checkpointing.dir = "checkpoint"
}
}
@@ -159,6 +163,8 @@ base_config: |
base.url = "wasbs://"${job.statebackend.blob.storage.container}"@"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.checkpointing.dir}
{% elif checkpoint_store_type == "s3" %}
base.url = "s3://"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.container}"/"${job.statebackend.blob.storage.checkpointing.dir}
+{% elif checkpoint_store_type == "gcloud" %}
+ base.url = "gs://"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.container}"/"${job.statebackend.blob.storage.checkpointing.dir}
{% endif %}
}
}
@@ -188,6 +194,7 @@ base_config: |
host = {{ postgres.db_url }}
port = {{ postgres.db_port }}
maxConnections = {{ postgres_max_connections }}
+ sslMode = {{ postgres.dp_ssl }}
user = "{{ postgres.db_username }}"
password = "{{ postgres.db_password }}"
}
@@ -236,7 +243,6 @@ ingest-router:
taskmanager.memory.process.size: {{ flink_job_names['ingest-router'].taskmanager_process_memory }}
jobmanager.memory.process.size: {{ flink_job_names['ingest-router'].jobmanager_process_memory }}
-
telemetry-extractor:
telemetry-extractor: |+
include file("/data/flink/conf/base-config.conf")
@@ -287,6 +293,7 @@ telemetry-extractor:
taskmanager.memory.process.size: {{ flink_job_names['telemetry-extractor'].taskmanager_process_memory }}
jobmanager.memory.process.size: {{ flink_job_names['telemetry-extractor'].jobmanager_process_memory }}
+
pipeline-preprocessor:
pipeline-preprocessor: |+
include file("/data/flink/conf/base-config.conf")
@@ -588,7 +595,13 @@ device-profile-updater:
port = {{ redis_meta_device_port }}
}
postgres {
+
+{% if postgres.dp_ssl_mode == "require" %}
+ database = "{{ postgres.dp_db_name }}",
+{% else %}
database = "{{ postgres.db_name }}",
+{% endif %}
+
table = "{{ device_profile_table }}"
}
diff --git a/kubernetes/helm_charts/druid-cluster/values.j2 b/kubernetes/helm_charts/druid-cluster/values.j2
index 169ef07b8e..bacc6bc321 100644
--- a/kubernetes/helm_charts/druid-cluster/values.j2
+++ b/kubernetes/helm_charts/druid-cluster/values.j2
@@ -24,8 +24,8 @@ druid_metadata_storage_connector_password: {{ druid_configs[cluster_type].druid_
# Druid Storage Type
druid_deepstorage_type: azure
-druid_azure_storage_account: "{{ sunbird_private_storage_account_name }}"
-druid_azure_storage_account_key: "{{ sunbird_private_storage_account_key }}"
+druid_azure_storage_account: "{{ cloud_private_storage_accountname }}"
+druid_azure_storage_account_key: "{{ cloud_private_storage_secret }}"
druid_azure_container: "{{ druid_configs[cluster_type].azure_container }}"
# Indexing service logs
diff --git a/kubernetes/helm_charts/secor/config/secor.common.properties b/kubernetes/helm_charts/secor/config/secor.common.properties
index 7050ebcf1b..fc207e660c 100644
--- a/kubernetes/helm_charts/secor/config/secor.common.properties
+++ b/kubernetes/helm_charts/secor/config/secor.common.properties
@@ -23,7 +23,7 @@ secor.kafka.topic_blacklist=
# Choose what to fill according to the service you are using
# in the choice option you can fill S3, GS, Swift or Azure
-cloud.service=Azure
+cloud.service={{ $.Values.storage_type }}
# AWS authentication credentials.
# Leave empty if using IAM role-based authentication with s3a filesystem.
@@ -56,7 +56,7 @@ aws.endpoint=
# Toggle the AWS S3 client between virtual host style access and path style
# access. See http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html
-aws.client.pathstyleaccess=false
+aws.client.pathstyleaccess=
###########################
# START AWS S3 ENCRYPTION #
@@ -357,7 +357,8 @@ secor.max.message.size.bytes=100000
# Class that will manage uploads. Default is to use the hadoop
# interface to S3.
-secor.upload.manager.class=com.pinterest.secor.uploader.AzureUploadManager
+# secor.upload.manager.class=com.pinterest.secor.uploader.AzureUploadManager
+secor.upload.manager.class=
#Set below property to your timezone, and the events will be parsed and converted to the timezone specified
secor.message.timezone=UTC
diff --git a/kubernetes/helm_charts/secor/config/secor.partition.properties b/kubernetes/helm_charts/secor/config/secor.partition.properties
index 743e1bab86..f14c523a5c 100644
--- a/kubernetes/helm_charts/secor/config/secor.partition.properties
+++ b/kubernetes/helm_charts/secor/config/secor.partition.properties
@@ -14,7 +14,12 @@
# limitations under the License.
include=secor.properties
+{{- if eq .Values.storage_type "Azure" }}
include=secor.azure.properties
+{{- end }}
+{{- if eq .Values.storage_type "S3" }}
+include=secor.s3.properties
+{{- end }}
# Name of the Kafka consumer group.
secor.kafka.group={{ get (get $.Values.secor_jobs $.Release.Name) "consumer_group" }}
@@ -62,6 +67,6 @@ secor.max.file.age.policy=oldest
# currentTime - Time of upload in HH-mm format
# currentDate - Time of upload in YYYYMMDD format
# folder - Folder to use based on message id map lookup
-secor.s3.output_file_pattern={{ get (get $.Values.secor_jobs $.Release.Name) "output_file_pattern" }}
+secor.s3.output_file_pattern=
secor.partition.message.channel.identifier={{ get (get $.Values.secor_jobs $.Release.Name) "message_channel_identifier" }}
diff --git a/kubernetes/helm_charts/secor/config/secor.s3.properties b/kubernetes/helm_charts/secor/config/secor.s3.properties
new file mode 100644
index 0000000000..e52e372f50
--- /dev/null
+++ b/kubernetes/helm_charts/secor/config/secor.s3.properties
@@ -0,0 +1,57 @@
+include=secor.properties
+
+
+
+############
+# MUST SET #
+############
+
+# Name of the s3 bucket where log files are stored.
+secor.s3.bucket={{ $.Values.s3_bucket_name }}
+
+# AWS authentication credentials.
+# Leave empty if using IAM role-based authentication with s3a filesystem.
+aws.access.key={{ $.Values.s3_access_key }}
+aws.secret.key={{ $.Values.s3_secret_id }}
+
+# AWS region or endpoint. region should be a known region name (eg.
+# us-east-1). endpoint should be a known S3 endpoint url. If neither
+# are specified, then the default region (us-east-1) is used. If both
+# are specified then endpoint is used.
+#
+# Only apply if the the S3UploadManager is used - see
+# secor.upload.manager.class.
+#
+# http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
+aws.region={{ $.Values.s3_region }}
+aws.endpoint={{ $.Values.s3_endpoint }}
+
+# Toggle the AWS S3 client between virtual host style access and path style
+# access. See http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html
+aws.client.pathstyleaccess={{ $.Values.s3_path_style_access }}
+
+# Class that will manage uploads. Default is to use the hadoop
+# interface to S3.
+# secor.upload.manager.class=com.pinterest.secor.uploader.AzureUploadManager
+secor.upload.manager.class=com.pinterest.secor.uploader.S3UploadManager
+
+# S3 path where sequence files are stored.
+secor.s3.path={{- get (get $.Values.secor_jobs $.Release.Name) "base_path" }}
+
+# Output file pattern excluding prefix. Defaults to topic/partition/generation_kafkaPartition_fmOffset.gz.
+# Available placeholders are
+# topic - The topic name the data is being fetched
+# partition - The partition name
+# generation - Generation
+# kafkaPartition - The kafka partition
+# fmOffset - First Message offset in the file.
+# randomHex - A 4 character random hex to append to the file name
+# currentTimestamp - Time of upload in epoch format
+# currentTime - Time of upload in HH-mm format
+# currentDate - Time of upload in YYYYMMDD format
+# folder - Folder to use based on message id map lookup
+secor.s3.output_file_pattern={{ get (get $.Values.secor_jobs $.Release.Name) "output_file_pattern" }}
+
+################
+# END MUST SET #
+################
\ No newline at end of file
diff --git a/kubernetes/helm_charts/secor/values.j2 b/kubernetes/helm_charts/secor/values.j2
index 4aa2e0ee83..7d9aa8ecec 100644
--- a/kubernetes/helm_charts/secor/values.j2
+++ b/kubernetes/helm_charts/secor/values.j2
@@ -1,9 +1,25 @@
-azure_account: "{{ sunbird_private_storage_account_name }}"
-azure_secret: "{{ sunbird_private_storage_account_key }}"
+azure_account: "{{ cloud_private_storage_accountname }}"
+azure_secret: "{{ cloud_private_storage_secret }}"
azure_container_name: "telemetry-data-store"
-namespace: {{ secor_namespace }}
+s3_access_key: "{{s3_storage_key}}"
+s3_secret_id: "{{s3_storage_secret}}"
+s3_region: "{{oci_region}}"
+s3_endpoint: "{{s3_storage_endpoint}}"
+s3_path_style_access: "{{s3_path_style_access}}"
+s3_bucket_name: "telemetry-data-store"
+
+{% if cloud_service_provider == 'oci' -%}
+storage_type: "S3"
+storageClass: "oci-bv"
+{%- else -%}
+storage_type: "Azure"
storageClass: {{ secor_storage_class | default('default') }}
+{%- endif %}
+
+
+
+namespace: {{ secor_namespace }}
imagepullsecrets: {{ imagepullsecrets }}
secor_jobs:
diff --git a/pipelines/deploy/spark-cluster-deploy/Jenkinsfile b/pipelines/deploy/spark-cluster-deploy/Jenkinsfile
index 9749d35b36..926c773ff0 100644
--- a/pipelines/deploy/spark-cluster-deploy/Jenkinsfile
+++ b/pipelines/deploy/spark-cluster-deploy/Jenkinsfile
@@ -26,7 +26,7 @@ node() {
jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
currentWs = sh(returnStdout: true, script: 'pwd').trim()
ansiblePlaybook = "${currentWs}/ansible/spark-cluster-job-submit.yml"
- ansibleExtraArgs = "--tags ${params.job_type} --extra-vars \"job_id=${params.job_id} mode=${params.mode} partitions=${params.partitions} parallelisation=${params.parallelisation} start_date=${params.start_date} end_date=${params.end_date} batch_id=${params.batch_identifier} sparkMaster=${params.sparkMaster} pause_min=${params.pause_min} selected_partitions=${params.selected_partitions}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass -vvvv "
+ ansibleExtraArgs = "--tags ${params.job_type} --extra-vars \"job_id=${params.job_id} mode=${params.mode} partitions=${params.partitions} parallelisation=${params.parallelisation} start_date=${params.start_date} end_date=${params.end_date} batch_id=${params.batch_identifier} sparkMaster=${params.sparkMaster} pause_min=${params.pause_min} vcn_name=${params.vcn_name} bds_cluster_name=${params.bds_cluster_name} selected_partitions=${params.selected_partitions}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass -vvvv "
values.put('currentWs', currentWs)
values.put('env', envDir)
values.put('module', module)
diff --git a/pipelines/deploy/spark-cluster-deploy/Jenkinsfile.parallel b/pipelines/deploy/spark-cluster-deploy/Jenkinsfile.parallel
index 4b9891d62a..480e880609 100644
--- a/pipelines/deploy/spark-cluster-deploy/Jenkinsfile.parallel
+++ b/pipelines/deploy/spark-cluster-deploy/Jenkinsfile.parallel
@@ -26,7 +26,7 @@ node() {
jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
currentWs = sh(returnStdout: true, script: 'pwd').trim()
ansiblePlaybook = "${currentWs}/ansible/spark-cluster-job-submit.yml"
- ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass --extra-vars \"jobs=${params.jobs_to_submit} batch_size=${params.batch_size}\" --tags config-update,${params.jobs_submit_type} -vvvv "
+ ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass --extra-vars \"jobs=${params.jobs_to_submit} oci_install_loc=${params.oci_install_loc} batch_size=${params.batch_size}\" --tags config-update,${params.jobs_submit_type}"
values.put('currentWs', currentWs)
values.put('env', envDir)
values.put('module', module)
diff --git a/pipelines/provision/spark/Jenkinsfile.bds b/pipelines/provision/spark/Jenkinsfile.bds
new file mode 100644
index 0000000000..418d5ded0a
--- /dev/null
+++ b/pipelines/provision/spark/Jenkinsfile.bds
@@ -0,0 +1,57 @@
+
+@Library('deploy-conf') _
+node('build-slave') {
+ try {
+ String ANSI_GREEN = "\u001B[32m"
+ String ANSI_NORMAL = "\u001B[0m"
+ String ANSI_BOLD = "\u001B[1m"
+ String ANSI_RED = "\u001B[31m"
+ String ANSI_YELLOW = "\u001B[33m"
+
+ ansiColor('xterm') {
+ stage('Checkout') {
+ checkout scm
+ }
+
+ stage('copy cluster creation script') {
+ values = [:]
+ envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+ module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+ jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+ currentWs = sh(returnStdout: true, script: 'pwd').trim()
+ ansiblePlaybook = "${currentWs}/ansible/oci-bds-spark.provision.yml"
+ ansibleExtraArgs = "--extra-vars \"compartment_id=${params.compartment_id} head_node_cpu=${params.head_node_cpu} worker_node_cpu=${params.worker_node_cpu} head_node_memory=${params.head_node_memory} worker_node_memory=${params.worker_node_memory} key_alias=${params.key_alias} user_id=${params.user_id} bucket=${params.bucket} subnet_id=${params.subnet_id} display_name=${params.display_name} workernode=${params.workernode} public_key=${params.public_key} cluster_state=${params.cluster_state}\" --tags copy-script,copy-creation-script --vault-password-file /var/lib/jenkins/secrets/vault-pass"
+ values.put('currentWs', currentWs)
+ values.put('env', envDir)
+ values.put('module', module)
+ values.put('jobName', jobName)
+ values.put('ansiblePlaybook', ansiblePlaybook)
+ values.put('ansibleExtraArgs', ansibleExtraArgs)
+ println values
+ ansible_playbook_run(values)
+
+ }
+ stage('create and provision spark OCI BDS') {
+ oci_namespace=params.oci_namespace
+ bucket=params.bucket
+ withCredentials([usernamePassword(credentialsId: 'oci-bds-credential', passwordVariable: 'cluster_password', usernameVariable: 'ambari_user')]) {
+ sh '''
+ currentws=$(pwd)
+ ansibleplaybook="$currentws/ansible/oci-bds-spark.provision.yml"
+ cd /tmp
+ ./create-cluster.sh $ambari_user $cluster_password
+ export ANSIBLE_HOST_KEY_CHECKING=False
+ ansible-playbook -i $currentws/ansible/inventory/env $ansibleplaybook --extra-vars "oci_namespace=$oci_namespace bucket=$bucket" --tags spark-provision --vault-password-file /var/lib/jenkins/secrets/vault-pass
+ '''
+ }
+
+ }
+
+ }
+ }
+ catch (err) {
+ currentBuild.result = "FAILURE"
+ throw err
+ }
+
+}
diff --git a/pipelines/provision/spark/Jenkinsfile.delete b/pipelines/provision/spark/Jenkinsfile.delete
index 93aed171cb..ce5e0b19dd 100644
--- a/pipelines/provision/spark/Jenkinsfile.delete
+++ b/pipelines/provision/spark/Jenkinsfile.delete
@@ -1,3 +1,4 @@
+
@Library('deploy-conf') _
node('build-slave') {
try {
@@ -18,8 +19,8 @@ node('build-slave') {
module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
currentWs = sh(returnStdout: true, script: 'pwd').trim()
- ansiblePlaybook = "${currentWs}/ansible/azure-hdinsight-spark.provision.yml"
- ansibleExtraArgs = "--extra-vars \"azure_resource_group=${params.resource_group} subscription_id=${env.subscription_id} tenant_id=${env.tenant_id} cluster_state=${params.type}\" --tags copy-script --vault-password-file /var/lib/jenkins/secrets/vault-pass"
+ ansiblePlaybook = "${currentWs}/ansible/oci-bds-spark.provision.yml"
+ ansibleExtraArgs = "--extra-vars \"compartment_id=${params.compartment_id} key_alias=${params.key_alias} user_id=${params.user_id} subnet_id=${params.subnet_id} display_name=${params.display_name} public_key=${params.public_key} cluster_state=${params.cluster_state}\" --tags copy-script,copy-creation-script --vault-password-file /var/lib/jenkins/secrets/vault-pass"
values.put('currentWs', currentWs)
values.put('env', envDir)
values.put('module', module)
@@ -29,15 +30,16 @@ node('build-slave') {
println values
ansible_playbook_run(values)
}
- stage('delete spark HDinsight cluster') {
+ stage('create and provision spark OCI BDS') {
storage_container=params.storage_container
- withCredentials([usernamePassword(credentialsId: 'azure-service-principal', passwordVariable: 'sppass', usernameVariable: 'spuser')]) {
+ //withCredentials([usernamePassword(credentialsId: 'azure-service-principal', passwordVariable: 'sppass', usernameVariable: 'spuser')]) {
sh '''
+ currentws=$(pwd)
+ ansibleplaybook="$currentws/ansible/oci-bds-spark.provision.yml"
cd /tmp
- ./delete-cluster.sh $spuser $sppass
+ ./delete-cluster.sh
'''
- }
-
+ //}
}